From 457f1fe30b66c97e7a8c3bbaaefcba89ab2ea0c1 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 6 Oct 2016 15:51:07 -0400
Subject: [PATCH 01/64] Provide resource to allow clients to expand custom
 expressions

Clients may now format a string as if it were to be an expression for an attribute,
and have the server evaluate it using the same engine without passing through the
attribute engine.  This makes it easier, for example, to do nodeexec n1-n4 ipmitool -H {hardwaremanagement.manager}
---
 confluent_server/confluent/config/configmanager.py |  8 ++++++++
 confluent_server/confluent/core.py                 |  1 +
 .../confluent/plugins/configuration/attributes.py  | 14 ++++++++++++++
 3 files changed, 23 insertions(+)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index c5043ab9..63d8aa4a 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -761,6 +761,14 @@ class ConfigManager(object):
                                                    decrypt=self.decrypt)
         return nodeobj
 
+    def expand_attrib_expression(self, nodelist, expression):
+        if type(nodelist) in (unicode, str):
+            nodelist = (nodelist,)
+        for node in nodelist:
+            cfgobj = self._cfgstore['nodes'][node]
+            fmt = _ExpressionFormat(cfgobj, node)
+            yield (node, fmt.format(expression))
+
     def get_node_attributes(self, nodelist, attributes=(), decrypt=None):
         if decrypt is None:
             decrypt = self.decrypt
diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py
index 9aa0adaf..608e8206 100644
--- a/confluent_server/confluent/core.py
+++ b/confluent_server/confluent/core.py
@@ -122,6 +122,7 @@ def _init_core():
         'attributes': {
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
+            'expression': PluginRoute({'handler': 'attributes'}),
         },
         'boot': {
             'nextdevice': PluginRoute({
diff --git a/confluent_server/confluent/plugins/configuration/attributes.py b/confluent_server/confluent/plugins/configuration/attributes.py
index 9ee0f149..10aa3307 100644
--- a/confluent_server/confluent/plugins/configuration/attributes.py
+++ b/confluent_server/confluent/plugins/configuration/attributes.py
@@ -152,6 +152,20 @@ def update_nodegroup(group, element, configmanager, inputdata):
     return retrieve_nodegroup(group, element, configmanager, inputdata)
 
 
+def _expand_expression(nodes, configmanager, inputdata):
+    expression = inputdata.get_attributes(list(nodes)[0])
+    if type(expression) is dict:
+        expression = expression['expression']
+    if type(expression) is dict:
+        expression = expression['expression']
+    for expanded in configmanager.expand_attrib_expression(nodes, expression):
+        yield msg.KeyValueData({'value': expanded[1]}, expanded[0])
+
+
+def create(nodes, element, configmanager, inputdata):
+    if nodes is not None and element[-1] == 'expression':
+        return _expand_expression(nodes, configmanager, inputdata)
+
 def update_nodes(nodes, element, configmanager, inputdata):
     updatedict = {}
     for node in nodes:

From 05a66641651a437145ad326450877e82ac5d1797 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 6 Oct 2016 16:30:29 -0400
Subject: [PATCH 02/64] Add a noderun command

This command enables running various commands against the nodes.
While I was at it, fix permissions on some files in git.
---
 confluent_client/bin/nodeconsole   |  0
 confluent_client/bin/nodeeventlog  |  0
 confluent_client/bin/nodefirmware  |  0
 confluent_client/bin/nodeinventory |  0
 confluent_client/bin/nodelist      |  0
 confluent_client/bin/noderun       | 86 ++++++++++++++++++++++++++++++
 6 files changed, 86 insertions(+)
 mode change 100644 => 100755 confluent_client/bin/nodeconsole
 mode change 100644 => 100755 confluent_client/bin/nodeeventlog
 mode change 100644 => 100755 confluent_client/bin/nodefirmware
 mode change 100644 => 100755 confluent_client/bin/nodeinventory
 mode change 100644 => 100755 confluent_client/bin/nodelist
 create mode 100755 confluent_client/bin/noderun

diff --git a/confluent_client/bin/nodeconsole b/confluent_client/bin/nodeconsole
old mode 100644
new mode 100755
diff --git a/confluent_client/bin/nodeeventlog b/confluent_client/bin/nodeeventlog
old mode 100644
new mode 100755
diff --git a/confluent_client/bin/nodefirmware b/confluent_client/bin/nodefirmware
old mode 100644
new mode 100755
diff --git a/confluent_client/bin/nodeinventory b/confluent_client/bin/nodeinventory
old mode 100644
new mode 100755
diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist
old mode 100644
new mode 100755
diff --git a/confluent_client/bin/noderun b/confluent_client/bin/noderun
new file mode 100755
index 00000000..4a614c12
--- /dev/null
+++ b/confluent_client/bin/noderun
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import select
+import shlex
+import subprocess
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+
+argparser = optparse.OptionParser(
+    usage="Usage: %prog node commandexpression",
+    epilog="Expressions are the same as in attributes, e.g. "
+           "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+argparser.disable_interspersed_args()
+(options, args) = argparser.parse_args()
+if len(args) < 2:
+    argparser.print_help()
+    sys.exit(1)
+c = client.Command()
+cmdstr = " ".join(args[1:])
+
+nodeforpopen = {}
+popens = []
+for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                    {'expression': cmdstr}):
+    ex = exp['databynode']
+    for node in ex:
+        cmd = ex[node]['value'].encode('utf-8')
+        cmdv = shlex.split(cmd)
+        nopen = subprocess.Popen(
+            cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        popens.append(nopen)
+        nodeforpopen[nopen] = node
+
+all = set([])
+pipedesc = {}
+exitcode = 0
+for pop in popens:
+    node = nodeforpopen[pop]
+    pipedesc[pop.stdout] = { 'node': node, 'popen': pop, 'type': 'stdout'}
+    pipedesc[pop.stderr] = {'node': node, 'popen': pop, 'type': 'stderr'}
+    all.add(pop.stdout)
+    all.add(pop.stderr)
+rdy, _, _ = select.select(all, [], [], 10)
+while all and rdy:
+    for r in rdy:
+        data = r.readline()
+        desc = pipedesc[r]
+        if data:
+            node = desc['node']
+            if desc['type'] == 'stdout':
+                sys.stdout.write('{0}: {1}'.format(node,data))
+            else:
+                sys.stderr.write('{0}: {1}'.format(node, data))
+        else:
+            pop = desc['popen']
+            ret = pop.poll()
+            if ret is not None:
+                exitcode = exitcode | ret
+                all.discard(r)
+    if all:
+        rdy, _, _ = select.select(all, [], [], 10)
+sys.exit(exitcode)
\ No newline at end of file

From 92fa2bf4d95ee524da81b974378c460d15df5231 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 13 Oct 2016 11:08:05 -0400
Subject: [PATCH 03/64] Add a number of security headers

There are a number of headers security scanners expect.
Explicitly declare how strict browser should be with
responses.
---
 confluent_server/confluent/httpapi.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index 323d55fd..ec921939 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -369,7 +369,12 @@ def resourcehandler_backend(env, start_response):
     """Function to handle new wsgi requests
     """
     mimetype, extension = _pick_mimetype(env)
-    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-cache')]
+    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-cache'),
+               ('X-Content-Type-Options', 'nosniff'),
+               ('Content-Security-Policy', "default-src 'self'"),
+               ('X-XSS-Protection', '1'), ('X-Frame-Options', 'deny'),
+               ('Strict-Transport-Security', 'max-age=86400'),
+               ('X-Permitted-Cross-Domain-Policies', 'none')]
     reqbody = None
     reqtype = None
     if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:

From 004d40e7ca36dd150d115215f9134490c799f11f Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 30 Nov 2016 09:57:58 -0500
Subject: [PATCH 04/64] Implement more CSRF countermeasures

At some expense of convenience, make it difficult
for CSRF to succeed even against the API explorer
for most people.  User/Password will now only be
accepted on hitting enter in the address bar rather
than following any link.
---
 confluent_server/confluent/httpapi.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index 4551eb0f..9997b9a5 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -233,6 +233,17 @@ def _csrf_valid(env, session):
             # oblige the request and apply a new token to the
             # session
             session['csrftoken'] = util.randomstring(32)
+        elif 'HTTP_REFERER' in env:
+            # If there is a referrer, make sure it stays consistent
+            # across the session.  A change in referer is a bad thing
+            try:
+                referer = env['HTTP_REFERER'].split('/')[2]
+            except IndexError:
+                return False
+            if 'validreferer' not in session:
+                session['validreferer'] = referer
+            elif session['validreferer'] != referer:
+                return False
         return True
     # The session has CSRF protection enabled, only mark valid if
     # the client has provided an auth token and that token matches the
@@ -274,6 +285,10 @@ def _authorize_request(env, operation):
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
         if env['PATH_INFO'] == '/sessions/current/logout':
             return ('logout',)
+        # We do not allow a link into the api browser to come in with just
+        # username and password
+        if 'HTTP_REFERER' in env:
+            return {'code': 401}
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
         authdata = auth.check_user_passphrase(name, passphrase, element=None)
@@ -369,7 +384,7 @@ def resourcehandler_backend(env, start_response):
     """Function to handle new wsgi requests
     """
     mimetype, extension = _pick_mimetype(env)
-    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-cache'),
+    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-store'),
                ('X-Content-Type-Options', 'nosniff'),
                ('Content-Security-Policy', "default-src 'self'"),
                ('X-XSS-Protection', '1'), ('X-Frame-Options', 'deny'),

From bf8dff90f3d1ebe61b6ce7cb2eb14f365a34db0d Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 30 Nov 2016 10:04:25 -0500
Subject: [PATCH 05/64] Provide CSRF protection to logout resource

Though more annoying than harmful, guard against CSRF succeeding
against the logout resource.
---
 confluent_server/confluent/httpapi.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index 9997b9a5..07a34aa9 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -268,27 +268,27 @@ def _authorize_request(env, operation):
             sessionid = cc['confluentsessionid'].value
             sessid = sessionid
             if sessionid in httpsessions:
-                if env['PATH_INFO'] == '/sessions/current/logout':
-                    targets = []
-                    for mythread in httpsessions[sessionid]['inflight']:
-                        targets.append(mythread)
-                    for mythread in targets:
-                        eventlet.greenthread.kill(mythread)
-                    del httpsessions[sessionid]
-                    return ('logout',)
                 if _csrf_valid(env, httpsessions[sessionid]):
+                    if env['PATH_INFO'] == '/sessions/current/logout':
+                        targets = []
+                        for mythread in httpsessions[sessionid]['inflight']:
+                            targets.append(mythread)
+                        for mythread in targets:
+                            eventlet.greenthread.kill(mythread)
+                        del httpsessions[sessionid]
+                        return ('logout',)
                     httpsessions[sessionid]['expiry'] = time.time() + 90
                     name = httpsessions[sessionid]['name']
                     authdata = auth.authorize(
                         name, element=None,
                         skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
-        if env['PATH_INFO'] == '/sessions/current/logout':
-            return ('logout',)
         # We do not allow a link into the api browser to come in with just
         # username and password
         if 'HTTP_REFERER' in env:
             return {'code': 401}
+        if env['PATH_INFO'] == '/sessions/current/logout':
+            return ('logout',)
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
         authdata = auth.check_user_passphrase(name, passphrase, element=None)

From d2156f3d6756a5542d02c40923c6a41d5c93ae6d Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 30 Nov 2016 10:15:44 -0500
Subject: [PATCH 06/64] Move the anti-referrer login to only logout

This has no functional difference, just a cosmetic
difference that does not give the erroneous impression
a logout actually occurred.  This does mean that if a browser
disables cookies and uses the api explorer, there would
be an opportunity for a CSRF.
---
 confluent_server/confluent/httpapi.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index 07a34aa9..e4ccdd22 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -283,11 +283,13 @@ def _authorize_request(env, operation):
                         name, element=None,
                         skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
-        # We do not allow a link into the api browser to come in with just
-        # username and password
-        if 'HTTP_REFERER' in env:
-            return {'code': 401}
         if env['PATH_INFO'] == '/sessions/current/logout':
+            if 'HTTP_REFERER' in env:
+                # note that this doesn't actually do harm
+                # otherwise, but this way do not give appearance
+                # of something having a side effect if it has the smell
+                # of a CSRF
+                return {'code': 401}
             return ('logout',)
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)

From c016c55340d1567327ee9a9ea12d97f0f2e7221b Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 30 Nov 2016 11:42:50 -0500
Subject: [PATCH 07/64] Add HTTP/1.0 cache suppression

There is not a single client that will ever talk
to this service that would not support HTTP/1.1.
However, do this to satisfy a scanner.
---
 confluent_server/confluent/httpapi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index e4ccdd22..445d6be2 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -387,6 +387,7 @@ def resourcehandler_backend(env, start_response):
     """
     mimetype, extension = _pick_mimetype(env)
     headers = [('Content-Type', mimetype), ('Cache-Control', 'no-store'),
+               ('Pragma', 'no-cache'),
                ('X-Content-Type-Options', 'nosniff'),
                ('Content-Security-Policy', "default-src 'self'"),
                ('X-XSS-Protection', '1'), ('X-Frame-Options', 'deny'),

From cb8cd290229f9df9a8ca576a5c0105ccf33ccafd Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 1 Dec 2016 16:37:50 -0500
Subject: [PATCH 08/64] Add two more man page sources

---
 confluent_client/doc/man/confetty.ronn    | 37 +++++++++++++++++++++++
 confluent_client/doc/man/nodeconsole.ronn | 30 ++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 confluent_client/doc/man/confetty.ronn
 create mode 100644 confluent_client/doc/man/nodeconsole.ronn

diff --git a/confluent_client/doc/man/confetty.ronn b/confluent_client/doc/man/confetty.ronn
new file mode 100644
index 00000000..b7c4758b
--- /dev/null
+++ b/confluent_client/doc/man/confetty.ronn
@@ -0,0 +1,37 @@
+confetty(1) --- Interactive confluent client
+=================================================
+
+## SYNOPSIS
+
+`confetty`
+
+## DESCRIPTION
+
+**confetty** launches an interactive CLI session to the
+confluent service.  It provides a filesystem-like
+view of the confluent interface.  It is intended to
+be mostly an aid for developing client software, with
+day to day administration generally being easier with
+the various function specific commands.
+
+## COMMANDS
+
+The CLI may be navigated by shell commands and some other
+commands.
+
+* `cd`:
+  Change the location within the tree
+* `ls`:
+  List the elements within the current directory/tree
+* `show` **ELEMENT**, `cat` **ELEMENT**:
+  Display the result of reading a specific element (by full or relative path)
+* `unset` **ELEMENT** **ATTRIBUTE**
+  For an element with attributes, request to clear the value of the attribue
+* `set` **ELEMENT** **ATTRIBUTE**=**VALUE**
+  Set the specified attribute to the given value
+* `start` **ELEMENT**
+  Start a console session indicated by **ELEMENT** (e.g. /nodes/n1/console/session)
+* `rm` **ELEMENT**
+  Request removal of an element.  (e.g. rm events/hardware/log clears log from a node)
+
+
diff --git a/confluent_client/doc/man/nodeconsole.ronn b/confluent_client/doc/man/nodeconsole.ronn
new file mode 100644
index 00000000..c7e3eb4d
--- /dev/null
+++ b/confluent_client/doc/man/nodeconsole.ronn
@@ -0,0 +1,30 @@
+nodeconsole(1) -- Open a console to a confluent node
+=====================================================
+
+## SYNOPSIS
+`nodeconsole` `node`
+
+## DESCRIPTION
+
+**nodeconsole** opens an interactive console session to a given node.  This is the
+text or serial console of a system.  Exiting is done by hitting `Ctrl-e`, then `c`,
+ then `.`.  Note that console output by default is additionally logged to 
+`/var/log/confluent/consoles/`**NODENAME**.
+
+## ESCAPE SEQUENCE COMMANDS
+
+While connected to a console, a number of commands may be performed through escape
+sequences.  To begin an command escape sequence, hit `Ctrl-e`, then `c`.  The next
+keystroke will be interpreted as a command.  The following commands are available.
+
+* `.`:
+  Exit the session and return to the command prompt
+* `b`:
+  Send a break to the remote console when possible (some console plugins may not support this)
+* `o`:
+  Request confluent to disconnect and reconnect to console.  For example if there is suspicion
+  that the console has gone inoperable, but would work if reconnected.
+* `?`:
+  Get a list of supported commands
+* `<enter>`:
+  Abandon entering an escape sequence command

From 8eef064b9fb82379a04e2d090b936466d561b91f Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 2 Dec 2016 10:35:24 -0500
Subject: [PATCH 09/64] Provide more useful error on socket error

If a socket error occurred that wasn't -2, the client would get an error
about KeyError, which wouldn't be helpful.  raise the unhandled exception
to provide more informative data in case of an issue.
---
 confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index 64fbe900..0f7106de 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -344,6 +344,7 @@ class IpmiHandler(object):
             except socket.gaierror as ge:
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
+                raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
         self.ipmicmd.setup_confluent_keyhandler()
 

From e7bdb5ee7d8c18234c09ec5706411a3809e2b897 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 3 Jan 2017 16:00:27 -0500
Subject: [PATCH 10/64] Defer TLS key handler registration

Wait until the object is logged in before trying to set
the key handler.  It carries some prerequisite on talking
to the BMC, so it is better handled in the logged handler.
---
 confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index 0f7106de..b4205ccc 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -346,7 +346,6 @@ class IpmiHandler(object):
                     raise exc.TargetEndpointUnreachable(ge[1])
                 raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
-        self.ipmicmd.setup_confluent_keyhandler()
 
     bootdevices = {
         'optical': 'cd'
@@ -358,6 +357,7 @@ class IpmiHandler(object):
             self.error = response['error']
         else:
             self.loggedin = True
+            self.ipmicmd.setup_confluent_keyhandler()
         self._logevt.set()
 
     def handle_request(self):

From b3c28ad33e374d55697dfbaf4515346ff2f03a58 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 4 Jan 2017 09:17:07 -0500
Subject: [PATCH 11/64] Defer assigning of self.ipmicmd until login

Wait until logged in to assign.  This way we have confirmation that ipmicmd is viable before trying to use it.
---
 confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index b4205ccc..52fe4533 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -345,7 +345,6 @@ class IpmiHandler(object):
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
                 raise
-        self.ipmicmd = persistent_ipmicmds[(node, tenant)]
 
     bootdevices = {
         'optical': 'cd'
@@ -356,6 +355,7 @@ class IpmiHandler(object):
             self.broken = True
             self.error = response['error']
         else:
+            self.ipmicmd = ipmicmd
             self.loggedin = True
             self.ipmicmd.setup_confluent_keyhandler()
         self._logevt.set()

From 7bf8242aba825012766c1ff14d031ab435eafd87 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 4 Jan 2017 09:44:52 -0500
Subject: [PATCH 12/64] Assign self.ipmicmd in case we are reusing an existing
 object that is already logged.

---
 confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index 52fe4533..eebf1872 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -345,6 +345,7 @@ class IpmiHandler(object):
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
                 raise
+        self.ipmicmd = persistent_ipmicmds([node, tenant])
 
     bootdevices = {
         'optical': 'cd'

From 5ffc2c298b8cb3bd006b05a63bd63a4a6ba45d4c Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 4 Jan 2017 09:56:31 -0500
Subject: [PATCH 13/64] Correct mixup of parentheses and brackets

---
 confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index eebf1872..d929b2df 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -345,7 +345,7 @@ class IpmiHandler(object):
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
                 raise
-        self.ipmicmd = persistent_ipmicmds([node, tenant])
+        self.ipmicmd = persistent_ipmicmds[(node, tenant)]
 
     bootdevices = {
         'optical': 'cd'

From 8c13e738c08f0b65feb6bd98c1c076f582140310 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 6 Jan 2017 13:28:28 -0500
Subject: [PATCH 14/64] Make usage/help more consistent across the commands

Have every client command run argparse to get a chance at '-h'.  When
lacking arguments, always use print_help() to provide detail rather than usage.
---
 confluent_client/bin/nodeeventlog  | 13 +++++++------
 confluent_client/bin/nodefirmware  | 11 ++++++-----
 confluent_client/bin/nodehealth    |  9 ++++++---
 confluent_client/bin/nodeidentify  |  9 ++++++---
 confluent_client/bin/nodeinventory |  9 +++++----
 confluent_client/bin/nodelist      |  4 ++--
 confluent_client/bin/nodepower     | 14 ++++++++------
 confluent_client/bin/noderun       |  2 +-
 confluent_client/bin/nodesensors   |  6 +++---
 confluent_client/bin/nodesetboot   |  7 +++----
 10 files changed, 47 insertions(+), 37 deletions(-)

diff --git a/confluent_client/bin/nodeeventlog b/confluent_client/bin/nodeeventlog
index e95f31f1..fc7a68a1 100755
--- a/confluent_client/bin/nodeeventlog
+++ b/confluent_client/bin/nodeeventlog
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # limitations under the License.
 
 from datetime import datetime as dt
+import optparse
 import os
 import sys
 
@@ -26,13 +27,13 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] noderange (clear)")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> [clear]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 deletemode = False
diff --git a/confluent_client/bin/nodefirmware b/confluent_client/bin/nodefirmware
index 356d3363..1b51d6ec 100755
--- a/confluent_client/bin/nodefirmware
+++ b/confluent_client/bin/nodefirmware
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 path = os.path.dirname(os.path.realpath(__file__))
@@ -57,12 +58,12 @@ def printfirm(node, prefix, data):
     print('{0}: {1}: {2}'.format(node, prefix, version))
 
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange>\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 try:
     session = client.Command()
diff --git a/confluent_client/bin/nodehealth b/confluent_client/bin/nodehealth
index 75a7c6b0..294a73b4 100755
--- a/confluent_client/bin/nodehealth
+++ b/confluent_client/bin/nodehealth
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import codecs
+import optparse
 import os
 import sys
 
@@ -28,10 +29,12 @@ import confluent.client as client
 
 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write('Usage: {0} <noderange>\n'.format(sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 
diff --git a/confluent_client/bin/nodeidentify b/confluent_client/bin/nodeidentify
index 956827ce..17c397d9 100755
--- a/confluent_client/bin/nodeidentify
+++ b/confluent_client/bin/nodeidentify
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 
@@ -25,10 +26,12 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write('Usage: {0} <noderange> [on|off]\n'.format(sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 identifystate = None
diff --git a/confluent_client/bin/nodeinventory b/confluent_client/bin/nodeinventory
index 8fc9caec..83a44bd7 100755
--- a/confluent_client/bin/nodeinventory
+++ b/confluent_client/bin/nodeinventory
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 path = os.path.dirname(os.path.realpath(__file__))
@@ -69,12 +70,12 @@ def printerror(res, node=None):
             exitcode = 1
 
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
     noderange = sys.argv[1]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange>\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 try:
     session = client.Command()
diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist
index 14b3fc87..50245628 100755
--- a/confluent_client/bin/nodelist
+++ b/confluent_client/bin/nodelist
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ def attrrequested(attr, attrlist, seenattributes):
             return True
     return False
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [list of attributes")
+    usage="Usage: %prog [options] noderange [list of attributes]")
 argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
 (options, args) = argparser.parse_args()
diff --git a/confluent_client/bin/nodepower b/confluent_client/bin/nodepower
index 7dbb8722..6711aea0 100755
--- a/confluent_client/bin/nodepower
+++ b/confluent_client/bin/nodepower
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 
@@ -25,13 +26,14 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] noderange "
+          "([status|on|off|shutdown|boot|reset])")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> ([status|on|off|shutdown|boot|reset]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 setstate = None
diff --git a/confluent_client/bin/noderun b/confluent_client/bin/noderun
index 4a614c12..dbebd4b2 100755
--- a/confluent_client/bin/noderun
+++ b/confluent_client/bin/noderun
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/confluent_client/bin/nodesensors b/confluent_client/bin/nodesensors
index a086f4d3..3c9827a3 100755
--- a/confluent_client/bin/nodesensors
+++ b/confluent_client/bin/nodesensors
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ sensorcollections = {
 
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [sensor(s)")
+    usage="Usage: %prog [options] noderange ([sensor(s)])")
 argparser.add_option('-i', '--interval', type='float',
                      help='Interval to do repeated samples over')
 argparser.add_option('-n', '--numreadings', type='int',
@@ -60,7 +60,7 @@ if options.numreadings:
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 sensors = []
 for sensorgroup in args[1:]:
diff --git a/confluent_client/bin/nodesetboot b/confluent_client/bin/nodesetboot
index 9a26e1bf..bbe71d46 100755
--- a/confluent_client/bin/nodesetboot
+++ b/confluent_client/bin/nodesetboot
@@ -26,7 +26,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser()
+argparser = optparse.OptionParser(
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')
@@ -40,9 +41,7 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
 try:
     noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> [default|cd|network|setup|hd]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 bootdev = None
 if len(sys.argv) > 2:

From 99e97fe5c462c6a14321015284752b9670927383 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 6 Jan 2017 13:29:25 -0500
Subject: [PATCH 15/64] Add the noderun command to the manifest

The noderun command was not being packaged.  Correct the
oversight so that it will be in packages.
---
 confluent_client/setup.py.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_client/setup.py.tmpl b/confluent_client/setup.py.tmpl
index 9bcddfd3..3fe14bc6 100644
--- a/confluent_client/setup.py.tmpl
+++ b/confluent_client/setup.py.tmpl
@@ -10,6 +10,6 @@ setup(
     scripts=['bin/confetty', 'bin/nodeconsole', 'bin/nodeeventlog',
              'bin/nodefirmware', 'bin/nodehealth', 'bin/nodeidentify',
              'bin/nodeinventory', 'bin/nodelist', 'bin/nodepower',
-             'bin/nodesensors', 'bin/nodesetboot'],
+             'bin/nodesensors', 'bin/nodesetboot', 'bin/noderun'],
     data_files=[('/etc/profile.d', ['confluent_env.sh'])],
 )

From 548e4404ce9b769568e99d296c00411c570f73b8 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 6 Jan 2017 13:43:02 -0500
Subject: [PATCH 16/64] Fix confluent startup error if dead

Clean up stale dbg.sock if present rather than fail to start.  Also, if the pid indicated in pid file doesn't exist, ignore
the pidfile contents.
---
 confluent_server/confluent/main.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py
index b8536eec..6e4ca9d6 100644
--- a/confluent_server/confluent/main.py
+++ b/confluent_server/confluent/main.py
@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -95,9 +95,15 @@ def _checkpidfile():
         fcntl.flock(pidfile, fcntl.LOCK_EX)
         pid = pidfile.read()
         if pid != '':
-            print ('/var/run/confluent/pid exists and indicates %s is still '
-                   'running' % pid)
-            sys.exit(1)
+            try:
+                os.kill(int(pid), 0)
+                print ('/var/run/confluent/pid exists and indicates %s is still '
+                       'running' % pid)
+                sys.exit(1)
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        pidfile.seek(0)
         pidfile.write(str(os.getpid()))
         fcntl.flock(pidfile, fcntl.LOCK_UN)
         pidfile.close()
@@ -199,10 +205,12 @@ def run():
     auth.init_auth()
     signal.signal(signal.SIGINT, terminate)
     signal.signal(signal.SIGTERM, terminate)
-    #TODO(jbjohnso): eventlet has a bug about unix domain sockets, this code
-    #works with bugs fixed
     if dbgif:
         oumask = os.umask(0077)
+        try:
+            os.remove('/var/run/confluent/dbg.sock')
+        except OSError:
+            pass  # We are not expecting the file to exist
         dbgsock = eventlet.listen("/var/run/confluent/dbg.sock",
                                    family=socket.AF_UNIX)
         eventlet.spawn_n(backdoor.backdoor_server, dbgsock)

From 486c322233ecc9e174f3defafb39a68fb4f0dcf1 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 17 Jan 2017 13:59:22 -0500
Subject: [PATCH 17/64] Have binding network sockets occur in a retry loop

There seems to be scenarios where a previously used socket won't open up immediately.  Retry when
this is detected.
---
 confluent_server/confluent/httpapi.py | 18 +++++++++++++++---
 confluent_server/confluent/sockapi.py | 11 ++++++++++-
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index 445d6be2..9e7ed2df 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -35,6 +35,7 @@ import eventlet.greenthread
 import greenlet
 import json
 import socket
+import sys
 import traceback
 import time
 import urlparse
@@ -741,9 +742,20 @@ def serve(bind_host, bind_port):
     #but deps are simpler without flup
     #also, the potential for direct http can be handy
     #todo remains unix domain socket for even http
-    eventlet.wsgi.server(
-        eventlet.listen((bind_host, bind_port, 0, 0), family=socket.AF_INET6),
-        resourcehandler, log=False, log_output=False, debug=False)
+    sock = None
+    while not sock:
+        try:
+            sock = eventlet.listen(
+                (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write(
+                'Failed to open HTTP due to busy port, trying again in'
+                ' a second\n')
+            eventlet.sleep(1)
+    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                         debug=False)
 
 
 class HttpApi(object):
diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py
index 874372d0..8205d7e3 100644
--- a/confluent_server/confluent/sockapi.py
+++ b/confluent_server/confluent/sockapi.py
@@ -244,7 +244,16 @@ def _tlshandler(bind_host, bind_port):
     plainsocket = socket.socket(socket.AF_INET6)
     plainsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     plainsocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-    plainsocket.bind((bind_host, bind_port, 0, 0))
+    bound = False
+    while not bound:
+        try:
+            plainsocket.bind((bind_host, bind_port, 0, 0))
+            bound = True
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write('TLS Socket in use, retrying in 1 second\n')
+            eventlet.sleep(1)
     plainsocket.listen(5)
     while (1):  # TODO: exithook
         cnn, addr = plainsocket.accept()

From bc5efa8a7eb53135546884708cdcabbaffb75387 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 20 Jan 2017 15:50:12 -0500
Subject: [PATCH 18/64] Try restarting confluent if running on update

This mitigates chance of confluent being stale.
---
 confluent_server/confluent_server.spec.tmpl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl
index 315daea9..6a8c7aec 100644
--- a/confluent_server/confluent_server.spec.tmpl
+++ b/confluent_server/confluent_server.spec.tmpl
@@ -33,6 +33,9 @@ done
 grep -v confluent/__init__.py INSTALLED_FILES.bare > INSTALLED_FILES
 cat INSTALLED_FILES
 
+%post
+if [ -x /usr/bin/systemctl ]; then /usr/bin/systemctl try-restart confluent; fi
+
 %clean
 rm -rf $RPM_BUILD_ROOT
 

From eca18a451bbd85e1e6272c854c703613bad06d58 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 10:09:15 -0500
Subject: [PATCH 19/64] Fix no password backup

If password=None was supplied, it would fail.  Now accomodate unprotected keys.
---
 .../confluent/config/configmanager.py             | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 63d8aa4a..4506e5e9 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1344,13 +1344,24 @@ class ConfigManager(object):
                 self._recalculate_expressions(cfgobj[key], formatter, node,
                                               changeset)
 
+
 def _dump_keys(password):
     if _masterkey is None or _masterintegritykey is None:
         init_masterkey()
     cryptkey = _format_key(_masterkey, password=password)
-    cryptkey = '!'.join(map(base64.b64encode, cryptkey['passphraseprotected']))
+    if 'passphraseprotected' in cryptkey:
+        cryptkey = '!'.join(map(base64.b64encode,
+                                cryptkey['passphraseprotected']))
+    else:
+        cryptkey = '*unencrypted:{0}'.format(base64.b64encode(
+            cryptkey['unencryptedvalue']))
     integritykey = _format_key(_masterintegritykey, password=password)
-    integritykey = '!'.join(map(base64.b64encode, integritykey['passphraseprotected']))
+    if 'passphraseprotected' in integritykey:
+        integritykey = '!'.join(map(base64.b64encode,
+                                    integritykey['passphraseprotected']))
+    else:
+        integritykey = '*unencrypted:{0}'.format(base64.b64encode(
+            integritykey['unencryptedvalue']))
     return json.dumps({'cryptkey': cryptkey, 'integritykey': integritykey},
                       sort_keys=True, indent=4, separators=(',', ': '))
 

From d2e8fa652139321ef086c93ec0bfb13c053e5d34 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 11:20:55 -0500
Subject: [PATCH 20/64] Start work on a db restore

Start by parsing the previously dumped key data, since the key data requires special handling.
---
 .../confluent/config/configmanager.py         | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 4506e5e9..8ce69753 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -128,6 +128,18 @@ def _get_protected_key(keydict, password, paramname):
         raise exc.LockedCredentials("No available decryption key")
 
 
+def _parse_key(keydata, password=None):
+    if keydata.startswith('*unencrypted:'):
+        return base64.b64decode(keydata[13:])
+    elif password:
+        salt, iv, crypt, hmac = [base64.b64decode(x)
+                                 for x in keydata.split('!')]
+        privkey, integkey = _derive_keys(password, salt)
+        return decrypt_value([iv, crypt, hmac], privkey, integkey)
+    raise(exc.LockedCredentials(
+        "Passphrase protected secret requires password"))
+
+
 def _format_key(key, password=None):
     if password is not None:
         salt = os.urandom(32)
@@ -1345,6 +1357,18 @@ class ConfigManager(object):
                                               changeset)
 
 
+def _restore_keys(jsond, password, newpassword):
+    # the jsond from the restored file, password (if any) used to protect
+    # the file, and newpassword to use, (also check the service.cfg file)
+    global _masterkey
+    global _masterintegritykey
+    keydata = json.loads(jsond)
+    cryptkey = _parse_key(keydata['cryptkey'], password)
+    integritykey = _parse_key(keydata['integritykey'], password)
+    _masterkey = cryptkey
+    _masterintegritykey = integritykey
+
+
 def _dump_keys(password):
     if _masterkey is None or _masterintegritykey is None:
         init_masterkey()

From d07e5a08c901fb55996e40e4f35615c07a35084b Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 14:17:29 -0500
Subject: [PATCH 21/64] Provide for a terminal session to process normal
 commands

If it is not one of the 'special' terminal ones, assume it's a normal one.  Recurse into
the process request to handle it as a separate request.
---
 confluent_server/confluent/sockapi.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py
index 8205d7e3..679c6140 100644
--- a/confluent_server/confluent/sockapi.py
+++ b/confluent_server/confluent/sockapi.py
@@ -170,7 +170,8 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
         auditlog.log(auditmsg)
     try:
         if operation == 'start':
-            return start_term(authname, cfm, connection, params, path)
+            return start_term(authname, cfm, connection, params, path,
+                              authdata, skipauth)
         elif operation == 'shutdown':
             configmanager.ConfigManager.shutdown()
         else:
@@ -187,7 +188,7 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
     return
 
 
-def start_term(authname, cfm, connection, params, path):
+def start_term(authname, cfm, connection, params, path, authdata, skipauth):
     elems = path.split('/')
     if len(elems) < 4 or elems[1] != 'nodes':
         raise exc.InvalidArgumentException('Invalid path {0}'.format(path))
@@ -233,7 +234,9 @@ def start_term(authname, cfm, connection, params, path):
                 consession.reopen()
                 continue
             else:
-                raise Exception("TODO")
+                process_request(connection, data, cfm, authdata, authname,
+                                skipauth)
+                continue
         if not data:
             consession.destroy()
             return

From 612350ca65ac6e3f7bfd38647acff85c9a6ce713 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 14:17:29 -0500
Subject: [PATCH 22/64] Provide for a terminal session to process normal
 commands

If it is not one of the 'special' terminal ones, assume it's a normal one.  Recurse into
the process request to handle it as a separate request.
---
 confluent_server/confluent/sockapi.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py
index 8205d7e3..679c6140 100644
--- a/confluent_server/confluent/sockapi.py
+++ b/confluent_server/confluent/sockapi.py
@@ -170,7 +170,8 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
         auditlog.log(auditmsg)
     try:
         if operation == 'start':
-            return start_term(authname, cfm, connection, params, path)
+            return start_term(authname, cfm, connection, params, path,
+                              authdata, skipauth)
         elif operation == 'shutdown':
             configmanager.ConfigManager.shutdown()
         else:
@@ -187,7 +188,7 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
     return
 
 
-def start_term(authname, cfm, connection, params, path):
+def start_term(authname, cfm, connection, params, path, authdata, skipauth):
     elems = path.split('/')
     if len(elems) < 4 or elems[1] != 'nodes':
         raise exc.InvalidArgumentException('Invalid path {0}'.format(path))
@@ -233,7 +234,9 @@ def start_term(authname, cfm, connection, params, path):
                 consession.reopen()
                 continue
             else:
-                raise Exception("TODO")
+                process_request(connection, data, cfm, authdata, authname,
+                                skipauth)
+                continue
         if not data:
             consession.destroy()
             return

From f245680732d35cc777a880572be8c92a8328c499 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 14:29:55 -0500
Subject: [PATCH 23/64] Rename nodeboot.py to nodeboot

No point in having '.py' extensions
---
 confluent_client/bin/{nodeboot.py => nodeboot} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename confluent_client/bin/{nodeboot.py => nodeboot} (100%)

diff --git a/confluent_client/bin/nodeboot.py b/confluent_client/bin/nodeboot
similarity index 100%
rename from confluent_client/bin/nodeboot.py
rename to confluent_client/bin/nodeboot

From 1028f1cb60ed19330fdbd353fd2027b0290eb65c Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 14:48:09 -0500
Subject: [PATCH 24/64] Have nodeboot be executable

---
 confluent_client/bin/nodeboot | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 confluent_client/bin/nodeboot

diff --git a/confluent_client/bin/nodeboot b/confluent_client/bin/nodeboot
old mode 100644
new mode 100755

From 15c84e8a9ba9f55181184a40719d037c021c21fd Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 15:23:16 -0500
Subject: [PATCH 25/64] Modify setup.py.tmpl to be adaptive

This should prevent forgetting to add content to
setup.py moving forward.
---
 confluent_client/setup.py.tmpl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/confluent_client/setup.py.tmpl b/confluent_client/setup.py.tmpl
index 3fe14bc6..f8768bd5 100644
--- a/confluent_client/setup.py.tmpl
+++ b/confluent_client/setup.py.tmpl
@@ -1,4 +1,7 @@
 from setuptools import setup
+import os
+
+scriptlist = ['bin/{0}'.format(d) for d in os.listdir('bin/')]
 
 setup(
     name='confluent_client',
@@ -7,9 +10,6 @@ setup(
     author_email='jjohnson2@lenovo.com',
     url='http://xcat.sf.net/',
     packages=['confluent'],
-    scripts=['bin/confetty', 'bin/nodeconsole', 'bin/nodeeventlog',
-             'bin/nodefirmware', 'bin/nodehealth', 'bin/nodeidentify',
-             'bin/nodeinventory', 'bin/nodelist', 'bin/nodepower',
-             'bin/nodesensors', 'bin/nodesetboot', 'bin/noderun'],
+    scripts=scriptlist,
     data_files=[('/etc/profile.d', ['confluent_env.sh'])],
 )

From 85205867b3ace1de030d76dbb9ca0f852bdf7487 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 16:50:03 -0500
Subject: [PATCH 26/64] Flesh out the restoration of the master keys

With this commit, the key portion of import should be complete.
---
 .../confluent/config/configmanager.py             | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 8ce69753..e00d79e1 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -65,6 +65,7 @@ import anydbm as dbm
 import ast
 import base64
 import confluent.config.attributes as allattributes
+import confluent.config.conf as conf
 import confluent.log
 import confluent.util
 import confluent.exceptions as exc
@@ -1357,7 +1358,7 @@ class ConfigManager(object):
                                               changeset)
 
 
-def _restore_keys(jsond, password, newpassword):
+def _restore_keys(jsond, password, newpassword=None):
     # the jsond from the restored file, password (if any) used to protect
     # the file, and newpassword to use, (also check the service.cfg file)
     global _masterkey
@@ -1365,8 +1366,20 @@ def _restore_keys(jsond, password, newpassword):
     keydata = json.loads(jsond)
     cryptkey = _parse_key(keydata['cryptkey'], password)
     integritykey = _parse_key(keydata['integritykey'], password)
+    conf.init_config()
+    cfg = conf.get_config()
+    if cfg.has_option('security', 'externalcfgkey'):
+        keyfilename = cfg.get('security', 'externalcfgkey')
+        with open(keyfilename, 'r') as keyfile:
+            newpassword = keyfile.read()
+    set_global('master_privacy_key', _format_key(cryptkey,
+                                                 password=newpassword))
+    set_global('master_integrity_key', _format_key(integritykey,
+                                                   password=newpassword))
     _masterkey = cryptkey
     _masterintegritykey = integritykey
+    ConfigManager.wait_for_sync()
+    # At this point, we should have the key situation all sorted
 
 
 def _dump_keys(password):

From f1867f900df110f6316f84862e7fcd01b591b492 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 15:31:36 -0500
Subject: [PATCH 27/64] Implement restore from db

This provides at least restore for the primary tenant (the only tenant currently used by anyone)
---
 .../confluent/config/configmanager.py         | 79 ++++++++++++++++++-
 1 file changed, 76 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index e00d79e1..7bf757be 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -720,6 +720,8 @@ class ConfigManager(object):
         :param uid: Custom identifier number if desired.  Defaults to random.
         :param displayname: Optional long format name for UI consumption
         """
+        if 'idmap' not in _cfgstore['main']:
+            _cfgstore['main']['idmap'] = {}
         if uid is None:
             uid = _generate_new_id()
         else:
@@ -733,8 +735,6 @@ class ConfigManager(object):
         self._cfgstore['users'][name] = {'id': uid}
         if displayname is not None:
             self._cfgstore['users'][name]['displayname'] = displayname
-        if 'idmap' not in _cfgstore['main']:
-            _cfgstore['main']['idmap'] = {}
         _cfgstore['main']['idmap'][uid] = {
             'tenant': self.tenant,
             'username': name
@@ -1200,6 +1200,68 @@ class ConfigManager(object):
         self._bg_sync_to_file()
         #TODO: wait for synchronization to suceed/fail??)
 
+    def _load_from_json(self, jsondata):
+        """Load fresh configuration data from jsondata
+
+        :param jsondata: String of jsondata
+        :return:
+        """
+        dumpdata = json.loads(jsondata)
+        tmpconfig = {}
+        for confarea in _config_areas:
+            if confarea not in dumpdata:
+                continue
+            tmpconfig[confarea] = {}
+            for element in dumpdata[confarea]:
+                newelement = copy.deepcopy(dumpdata[confarea][element])
+                for attribute in dumpdata[confarea][element]:
+                    if newelement[attribute] == '*REDACTED*':
+                        raise Exception(
+                            "Unable to restore from redacted backup")
+                    elif attribute == 'cryptpass':
+                        passparts = newelement[attribute].split('!')
+                        newelement[attribute] = tuple([base64.b64decode(x)
+                                                       for x in passparts])
+                    elif 'cryptvalue' in newelement[attribute]:
+                        bincrypt = newelement[attribute]['cryptvalue']
+                        bincrypt = tuple([base64.b64decode(x)
+                                          for x in bincrypt.split('!')])
+                        newelement[attribute]['cryptvalue'] = bincrypt
+                    elif attribute in ('nodes', '_expressionkeys'):
+                        # A group with nodes
+                        # delete it and defer until nodes are being added
+                        # which will implicitly fill this up
+                        # Or _expressionkeys attribute, which will similarly
+                        # be rebuilt
+                        del newelement[attribute]
+                tmpconfig[confarea][element] = newelement
+        # We made it through above section without an exception, go ahead and
+        # replace
+        # Start by erasing the dbm files if present
+        for confarea in _config_areas:
+            try:
+                os.unlink(os.path.join(self._cfgdir, confarea))
+            except OSError as e:
+                if e.errno == 2:
+                    pass
+        # Now we have to iterate through each fixed up element, using the
+        # set attribute to flesh out inheritence and expressions
+        for confarea in tmpconfig:
+            if confarea == 'nodes':
+                self.set_node_attributes(tmpconfig[confarea], True)
+            elif confarea == 'nodegroups':
+                self.set_group_attributes(tmpconfig[confarea], True)
+            elif confarea == 'users':
+                for user in tmpconfig[confarea]:
+                    uid = tmpconfig[confarea].get('id', None)
+                    displayname = tmpconfig[confarea].get('displayname', None)
+                    self.create_user(user, uid=uid, displayname=displayname)
+                    if 'cryptpass' in tmpconfig[confarea][user]:
+                        self._cfgstore['users'][user]['cryptpass'] = \
+                            tmpconfig[confarea][user]['cryptpass']
+                        _mark_dirtykey('users', user, self.tenant)
+        self._bg_sync_to_file()
+
     def _dump_to_json(self, redact=None):
         """Dump the configuration in json form to output
 
@@ -1403,6 +1465,16 @@ def _dump_keys(password):
                       sort_keys=True, indent=4, separators=(',', ': '))
 
 
+def restore_db_from_directory(location, password):
+    with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+        keydata = cfgfile.read()
+        json.loads(keydata)
+        _restore_keys(keydata, password)
+    with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
+        cfgdata = cfgfile.read()
+        ConfigManager(tenant=None)._load_from_json(cfgdata)
+
+
 def dump_db_to_directory(location, password, redact=None):
     with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
         cfgfile.write(_dump_keys(password))
@@ -1413,7 +1485,8 @@ def dump_db_to_directory(location, password, redact=None):
     try:
         for tenant in os.listdir(
                 os.path.join(ConfigManager._cfgdir, '/tenants/')):
-            with open(os.path.join(location, tenant + '.json'), 'w') as cfgfile:
+            with open(os.path.join(location, 'tenants', tenant,
+                                   'main.json'), 'w') as cfgfile:
                 cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
                     redact=redact))
                 cfgfile.write('\n')

From 4a4b1a623f1f1944ce6ad13b762e1d56fd0bf1db Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:08:28 -0500
Subject: [PATCH 28/64] Add a utility to frontend DB dump/restore

This exposes the library functions as a utility
---
 confluent_server/bin/confluentdbutil | 66 ++++++++++++++++++++++++++++
 confluent_server/confluent/main.py   | 20 +++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 confluent_server/bin/confluentdbutil

diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil
new file mode 100644
index 00000000..828f06da
--- /dev/null
+++ b/confluent_server/bin/confluentdbutil
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import optparse
+import sys
+import os
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    # if installed into system path, do not muck with things
+    sys.path.append(path)
+import confluent.config.configmanager as cfm
+import confluent.config.conf as conf
+import confluent.main as main
+
+argparser = optparse.OptionParser(usage="Usage: %prog [options] [dump|restore] [path]")
+argparser.add_option('-p', '--password',
+                     help='Password to use to protect/unlock a protected dump')
+argparser.add_option('-r', '--redact', action='store_true',
+                     help='Redact potentially sensitive data rather than store')
+argparser.add_option('-u', '--unprotected', action='store_true',
+                      help='Specify that no password should be used to protect'
+                           ' the key information.  Fields will be encrypted, '
+                           'but keys.json will contain unencrypted decryption'
+                           ' keys that may be used to read the dump')
+(options, args) = argparser.parse_args()
+if len(args) != 2 or args[0] not in ('dump', 'restore'):
+    argparser.print_help()
+    sys.exit(1)
+dumpdir = args[1]
+
+
+if args[0] == 'restore':
+    pid = main.is_running()
+    if pid is not None:
+        print("Confluent is running, must shut down to restore db")
+        sys.exit(1)
+    cfm.restore_db_from_directory(dumpdir, options.password)
+elif args[0] == 'dump':
+    if options.password is None and not options.unprotected:
+        print("Must indicate a password to protect or -u to opt opt of "
+              "secure value protection")
+        sys.exit(1)
+    os.umask(077)
+    main._initsecurity(conf.get_config())
+    if not os.path.exists(dumpdir):
+        os.makedirs(dumpdir)
+    cfm.dump_db_to_directory(dumpdir, options.password)
+
+
+
diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py
index 6e4ca9d6..2c5ce1d5 100644
--- a/confluent_server/confluent/main.py
+++ b/confluent_server/confluent/main.py
@@ -89,6 +89,26 @@ def _updatepidfile():
     pidfile.close()
 
 
+def is_running():
+    # Utility function for utilities to check if confluent is running
+    try:
+        pidfile = open('/var/run/confluent/pid', 'r+')
+        fcntl.flock(pidfile, fcntl.LOCK_SH)
+        pid = pidfile.read()
+        if pid != '':
+            try:
+                os.kill(int(pid), 0)
+                return pid
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        fcntl.flock(pidfile, fcntl.LOCK_UN)
+        pidfile.close()
+    except IOError:
+        pass
+    return None
+
+
 def _checkpidfile():
     try:
         pidfile = open('/var/run/confluent/pid', 'r+')

From cfc6fd04fcd29eea8b435c49cd8cc3c0aa2b90f7 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:10:01 -0500
Subject: [PATCH 29/64] Make confluentdbutil executable

---
 confluent_server/bin/confluentdbutil | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 confluent_server/bin/confluentdbutil

diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil
old mode 100644
new mode 100755

From cbc0ffbc1c1570fbfd8b0bbec37d36674a879c7b Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:12:49 -0500
Subject: [PATCH 30/64] Actually hook up the redact feature

---
 confluent_server/bin/confluentdbutil | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil
index 828f06da..e5acb419 100755
--- a/confluent_server/bin/confluentdbutil
+++ b/confluent_server/bin/confluentdbutil
@@ -28,7 +28,8 @@ import confluent.config.configmanager as cfm
 import confluent.config.conf as conf
 import confluent.main as main
 
-argparser = optparse.OptionParser(usage="Usage: %prog [options] [dump|restore] [path]")
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] [dump|restore] [path]")
 argparser.add_option('-p', '--password',
                      help='Password to use to protect/unlock a protected dump')
 argparser.add_option('-r', '--redact', action='store_true',
@@ -52,15 +53,15 @@ if args[0] == 'restore':
         sys.exit(1)
     cfm.restore_db_from_directory(dumpdir, options.password)
 elif args[0] == 'dump':
-    if options.password is None and not options.unprotected:
+    if options.password is None and not (options.unprotected or options.redact):
         print("Must indicate a password to protect or -u to opt opt of "
-              "secure value protection")
+              "secure value protection or -r to skip all protected data")
         sys.exit(1)
     os.umask(077)
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)
-    cfm.dump_db_to_directory(dumpdir, options.password)
+    cfm.dump_db_to_directory(dumpdir, options.password, options.redact)
 
 
 

From a288136a80d3ed517558fdecc8103e4f4783a046 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:13:49 -0500
Subject: [PATCH 31/64] Do not write out keys when redacting

It's silly to store keys when redact is requested
---
 confluent_server/confluent/config/configmanager.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 7bf757be..2876b9d0 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1476,9 +1476,10 @@ def restore_db_from_directory(location, password):
 
 
 def dump_db_to_directory(location, password, redact=None):
-    with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
-        cfgfile.write(_dump_keys(password))
-        cfgfile.write('\n')
+    if not redact:
+        with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
+            cfgfile.write(_dump_keys(password))
+            cfgfile.write('\n')
     with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
         cfgfile.write('\n')

From c381fefc494415768eb186ef0dd432dc3d02ff2c Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:27:49 -0500
Subject: [PATCH 32/64] Give a friendlier message on restore of redact DB

A redacted dump will not have a keys.json file, which
is natural.  Replace 'file not found' with a message
indicating the possibility of a redacted dump.
---
 confluent_server/confluent/config/configmanager.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 2876b9d0..10805ea7 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1466,10 +1466,15 @@ def _dump_keys(password):
 
 
 def restore_db_from_directory(location, password):
-    with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
-        keydata = cfgfile.read()
-        json.loads(keydata)
-        _restore_keys(keydata, password)
+    try:
+        with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+            keydata = cfgfile.read()
+            json.loads(keydata)
+            _restore_keys(keydata, password)
+    except IOError as e:
+        if e.errno == 2:
+            raise Exception("Cannot restore without keys, this may be a "
+                            "redacted dump")
     with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
         cfgdata = cfgfile.read()
         ConfigManager(tenant=None)._load_from_json(cfgdata)

From 3b38d8ac430d225f4836e0c879d8dee199783be6 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:38:43 -0500
Subject: [PATCH 33/64] Assure deterministic enumeration of areas

nodegroup must be restored first.
---
 confluent_server/confluent/config/configmanager.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 10805ea7..4a36a97d 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1246,7 +1246,9 @@ class ConfigManager(object):
                     pass
         # Now we have to iterate through each fixed up element, using the
         # set attribute to flesh out inheritence and expressions
-        for confarea in tmpconfig:
+        for confarea in _config_areas:
+            if confarea not in tmpconfig:
+                continue
             if confarea == 'nodes':
                 self.set_node_attributes(tmpconfig[confarea], True)
             elif confarea == 'nodegroups':

From 02dd29b027a4404e2a44945ffb882de5e6edeb8e Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 31 Jan 2017 15:38:54 -0500
Subject: [PATCH 34/64] Add the util to the setup.py

Previously packaging was missing the new utility
---
 confluent_server/setup.py.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl
index 9bd1d2ac..fdbbee84 100644
--- a/confluent_server/setup.py.tmpl
+++ b/confluent_server/setup.py.tmpl
@@ -5,7 +5,7 @@ setup(
     name='confluent_server',
     version='#VERSION#',
     author='Jarrod Johnson',
-    author_email='jbjohnso@us.ibm.com',
+    author_email='jjohnson2@lenovo.com',
     url='http://xcat.sf.net/',
     description='confluent systems management server',
     packages=['confluent', 'confluent/config', 'confluent/interface',
@@ -14,7 +14,7 @@ setup(
               'confluent/plugins/configuration/'],
     install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet',
                       'pyghmi>=0.6.5'],
-    scripts=['bin/confluent'],
+    scripts=['bin/confluent', 'bin/confluentdbutil'],
     data_files=[('/etc/init.d', ['sysvinit/confluent']),
                 ('/usr/lib/systemd/system', ['systemd/confluent.service']),
 		('/opt/confluent/lib/python/confluent/plugins/console/', [])],

From ffd1bdfae3ba454f437274ca24c7fb66a4510585 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 2 Feb 2017 17:00:09 -0500
Subject: [PATCH 35/64] Clean up nodepower

nodepower boot and reset output was misleading.
---
 confluent_client/bin/nodepower                   |  1 +
 confluent_client/confluent/client.py             | 11 ++++++++++-
 confluent_server/confluent/messages.py           |  5 +++++
 .../confluent/plugins/hardwaremanagement/ipmi.py | 16 ++++++++++++++--
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/confluent_client/bin/nodepower b/confluent_client/bin/nodepower
index 6711aea0..5dd0b007 100755
--- a/confluent_client/bin/nodepower
+++ b/confluent_client/bin/nodepower
@@ -45,5 +45,6 @@ if len(sys.argv) > 2:
 
 session = client.Command()
 exitcode = 0
+session.add_precede_key('oldstate')
 sys.exit(
     session.simple_noderange_command(noderange, '/power/state', setstate))
\ No newline at end of file
diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py
index 38ff2bbf..aa30e6a1 100644
--- a/confluent_client/confluent/client.py
+++ b/confluent_client/confluent/client.py
@@ -44,6 +44,7 @@ def _parseserver(string):
 class Command(object):
 
     def __init__(self, server=None):
+        self._prevkeyname = None
         self.connection = None
         if server is None:
             if 'CONFLUENT_HOST' in os.environ:
@@ -74,6 +75,9 @@ class Command(object):
         if authdata['authpassed'] == 1:
             self.authenticated = True
 
+    def add_precede_key(self, keyname):
+        self._prevkeyname = keyname
+
     def handle_results(self, ikey, rc, res):
         if 'error' in res:
             sys.stderr.write('Error: {0}\n'.format(res['error']))
@@ -93,7 +97,12 @@ class Command(object):
                 else:
                     rc |= 1
             elif ikey in res[node]:
-                print('{0}: {1}'.format(node, res[node][ikey]['value']))
+                if self._prevkeyname and self._prevkeyname in res[node]:
+                    print('{0}: {2}->{1}'.format(
+                        node, res[node][ikey]['value'],
+                        res[node][self._prevkeyname]['value']))
+                else:
+                    print('{0}: {1}'.format(node, res[node][ikey]['value']))
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
diff --git a/confluent_server/confluent/messages.py b/confluent_server/confluent/messages.py
index 4f43abe1..fc4a7df2 100644
--- a/confluent_server/confluent/messages.py
+++ b/confluent_server/confluent/messages.py
@@ -797,6 +797,11 @@ class PowerState(ConfluentChoiceMessage):
     ])
     keyname = 'state'
 
+    def __init__(self, node, state, oldstate=None):
+        super(PowerState, self).__init__(node, state)
+        if oldstate is not None:
+            self.kvpairs[node]['oldstate'] = {'value': oldstate}
+
 
 class BMCReset(ConfluentChoiceMessage):
     valid_values = set([
diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index d929b2df..ce92b869 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -795,10 +795,22 @@ class IpmiHandler(object):
             return
         elif 'update' == self.op:
             powerstate = self.inputdata.powerstate(self.node)
+            oldpower = None
+            if powerstate == 'boot':
+                oldpower = self.ipmicmd.get_power()
+                if 'powerstate' in oldpower:
+                    oldpower = oldpower['powerstate']
             self.ipmicmd.set_power(powerstate, wait=30)
-            power = self.ipmicmd.get_power()
+            if powerstate == 'boot' and oldpower == 'on':
+                power = {'powerstate': 'reset'}
+            else:
+                power = self.ipmicmd.get_power()
+                if powerstate == 'reset' and power['powerstate'] == 'on':
+                    power['powerstate'] = 'reset'
+
             self.output.put(msg.PowerState(node=self.node,
-                                           state=power['powerstate']))
+                                           state=power['powerstate'],
+                                           oldstate=oldpower))
             return
 
     def handle_reset(self):

From ffbe1ab1561ef29375c025e7c141128cc23defd9 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 10 Feb 2017 11:20:39 -0500
Subject: [PATCH 36/64] Protect against stale data in new requests

If a caller (reasonably) broke out of a loop, a subsequent call would get old data.
Protect against this by discarding data not consumed if previously called.
---
 confluent_client/confluent/client.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py
index aa30e6a1..f3c91225 100644
--- a/confluent_client/confluent/client.py
+++ b/confluent_client/confluent/client.py
@@ -27,6 +27,8 @@ import confluent.tlvdata as tlvdata
 SO_PASSCRED = 16
 
 
+inflight = False
+
 def _parseserver(string):
     if ']:' in string:
         server, port = string[1:].split(']:')
@@ -226,6 +228,15 @@ def send_request(operation, path, server, parameters=None):
     :param server: The socket to send data over
     :param parameters:  Parameters if any to send along with the request
     """
+    global inflight
+    if inflight:
+        # calling code failed to complete a transaction, flush and discard
+        # their unused data
+        result = tlvdata.recv(server)
+        while '_requestdone' not in result:
+            yield result
+            result = tlvdata.recv(server)
+    inflight = True
     payload = {'operation': operation, 'path': path}
     if parameters is not None:
         payload['parameters'] = parameters
@@ -234,6 +245,7 @@ def send_request(operation, path, server, parameters=None):
     while '_requestdone' not in result:
         yield result
         result = tlvdata.recv(server)
+    inflight = False
 
 
 

From 583e3474ac8bda4164cd432e1c1b819e8006e1d1 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 10 Feb 2017 11:37:56 -0500
Subject: [PATCH 37/64] Do not yield data to discard

---
 confluent_client/confluent/client.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py
index f3c91225..07a85423 100644
--- a/confluent_client/confluent/client.py
+++ b/confluent_client/confluent/client.py
@@ -234,7 +234,6 @@ def send_request(operation, path, server, parameters=None):
         # their unused data
         result = tlvdata.recv(server)
         while '_requestdone' not in result:
-            yield result
             result = tlvdata.recv(server)
     inflight = True
     payload = {'operation': operation, 'path': path}

From f46269a6f2cbf2587916d40e5970bfc873dbed89 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 10 Feb 2017 16:41:47 -0500
Subject: [PATCH 38/64] Add debug of broken expression to nodelist

If an expression is invalid, have nodelist give the error data
allowing the user to see and potentially take action.
---
 confluent_client/bin/nodelist | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist
index 50245628..6cff5dcd 100755
--- a/confluent_client/bin/nodelist
+++ b/confluent_client/bin/nodelist
@@ -76,7 +76,11 @@ if len(args) > 1:
                             attrout = '{0}: {1}: ********'.format(node, attr)
                         else:
                             attrout = '{0}: {1}:'.format(node, attr)
-                    if options.blame:
+                    elif 'broken' in currattr:
+                        attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                                  '{2}'.format(node, attr,
+                                               currattr['broken'])
+                    if options.blame or 'broken' in currattr:
                         blamedata = []
                         if 'inheritedfrom' in currattr:
                             blamedata.append('inherited from group {0}'.format(

From 638842beec81c99764f9c70ffc5b141bcefecdd6 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 10:09:15 -0500
Subject: [PATCH 39/64] Fix no password backup

If password=None was supplied, it would fail.  Now accomodate unprotected keys.
---
 .../confluent/config/configmanager.py             | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 63d8aa4a..4506e5e9 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1344,13 +1344,24 @@ class ConfigManager(object):
                 self._recalculate_expressions(cfgobj[key], formatter, node,
                                               changeset)
 
+
 def _dump_keys(password):
     if _masterkey is None or _masterintegritykey is None:
         init_masterkey()
     cryptkey = _format_key(_masterkey, password=password)
-    cryptkey = '!'.join(map(base64.b64encode, cryptkey['passphraseprotected']))
+    if 'passphraseprotected' in cryptkey:
+        cryptkey = '!'.join(map(base64.b64encode,
+                                cryptkey['passphraseprotected']))
+    else:
+        cryptkey = '*unencrypted:{0}'.format(base64.b64encode(
+            cryptkey['unencryptedvalue']))
     integritykey = _format_key(_masterintegritykey, password=password)
-    integritykey = '!'.join(map(base64.b64encode, integritykey['passphraseprotected']))
+    if 'passphraseprotected' in integritykey:
+        integritykey = '!'.join(map(base64.b64encode,
+                                    integritykey['passphraseprotected']))
+    else:
+        integritykey = '*unencrypted:{0}'.format(base64.b64encode(
+            integritykey['unencryptedvalue']))
     return json.dumps({'cryptkey': cryptkey, 'integritykey': integritykey},
                       sort_keys=True, indent=4, separators=(',', ': '))
 

From a91d7047b22247ab3b66d5b0d7e1d82f9432916a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 11:20:55 -0500
Subject: [PATCH 40/64] Start work on a db restore

Start by parsing the previously dumped key data, since the key data requires special handling.
---
 .../confluent/config/configmanager.py         | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 4506e5e9..8ce69753 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -128,6 +128,18 @@ def _get_protected_key(keydict, password, paramname):
         raise exc.LockedCredentials("No available decryption key")
 
 
+def _parse_key(keydata, password=None):
+    if keydata.startswith('*unencrypted:'):
+        return base64.b64decode(keydata[13:])
+    elif password:
+        salt, iv, crypt, hmac = [base64.b64decode(x)
+                                 for x in keydata.split('!')]
+        privkey, integkey = _derive_keys(password, salt)
+        return decrypt_value([iv, crypt, hmac], privkey, integkey)
+    raise(exc.LockedCredentials(
+        "Passphrase protected secret requires password"))
+
+
 def _format_key(key, password=None):
     if password is not None:
         salt = os.urandom(32)
@@ -1345,6 +1357,18 @@ class ConfigManager(object):
                                               changeset)
 
 
+def _restore_keys(jsond, password, newpassword):
+    # the jsond from the restored file, password (if any) used to protect
+    # the file, and newpassword to use, (also check the service.cfg file)
+    global _masterkey
+    global _masterintegritykey
+    keydata = json.loads(jsond)
+    cryptkey = _parse_key(keydata['cryptkey'], password)
+    integritykey = _parse_key(keydata['integritykey'], password)
+    _masterkey = cryptkey
+    _masterintegritykey = integritykey
+
+
 def _dump_keys(password):
     if _masterkey is None or _masterintegritykey is None:
         init_masterkey()

From b87cb87c2ab51815b01044e616375c6fabe78ea9 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 27 Jan 2017 16:50:03 -0500
Subject: [PATCH 41/64] Flesh out the restoration of the master keys

With this commit, the key portion of import should be complete.
---
 .../confluent/config/configmanager.py             | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 8ce69753..e00d79e1 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -65,6 +65,7 @@ import anydbm as dbm
 import ast
 import base64
 import confluent.config.attributes as allattributes
+import confluent.config.conf as conf
 import confluent.log
 import confluent.util
 import confluent.exceptions as exc
@@ -1357,7 +1358,7 @@ class ConfigManager(object):
                                               changeset)
 
 
-def _restore_keys(jsond, password, newpassword):
+def _restore_keys(jsond, password, newpassword=None):
     # the jsond from the restored file, password (if any) used to protect
     # the file, and newpassword to use, (also check the service.cfg file)
     global _masterkey
@@ -1365,8 +1366,20 @@ def _restore_keys(jsond, password, newpassword):
     keydata = json.loads(jsond)
     cryptkey = _parse_key(keydata['cryptkey'], password)
     integritykey = _parse_key(keydata['integritykey'], password)
+    conf.init_config()
+    cfg = conf.get_config()
+    if cfg.has_option('security', 'externalcfgkey'):
+        keyfilename = cfg.get('security', 'externalcfgkey')
+        with open(keyfilename, 'r') as keyfile:
+            newpassword = keyfile.read()
+    set_global('master_privacy_key', _format_key(cryptkey,
+                                                 password=newpassword))
+    set_global('master_integrity_key', _format_key(integritykey,
+                                                   password=newpassword))
     _masterkey = cryptkey
     _masterintegritykey = integritykey
+    ConfigManager.wait_for_sync()
+    # At this point, we should have the key situation all sorted
 
 
 def _dump_keys(password):

From 393ea41696a51101277a6fbcc99cd58529406093 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 15:31:36 -0500
Subject: [PATCH 42/64] Implement restore from db

This provides at least restore for the primary tenant (the only tenant currently used by anyone)
---
 .../confluent/config/configmanager.py         | 79 ++++++++++++++++++-
 1 file changed, 76 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index e00d79e1..7bf757be 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -720,6 +720,8 @@ class ConfigManager(object):
         :param uid: Custom identifier number if desired.  Defaults to random.
         :param displayname: Optional long format name for UI consumption
         """
+        if 'idmap' not in _cfgstore['main']:
+            _cfgstore['main']['idmap'] = {}
         if uid is None:
             uid = _generate_new_id()
         else:
@@ -733,8 +735,6 @@ class ConfigManager(object):
         self._cfgstore['users'][name] = {'id': uid}
         if displayname is not None:
             self._cfgstore['users'][name]['displayname'] = displayname
-        if 'idmap' not in _cfgstore['main']:
-            _cfgstore['main']['idmap'] = {}
         _cfgstore['main']['idmap'][uid] = {
             'tenant': self.tenant,
             'username': name
@@ -1200,6 +1200,68 @@ class ConfigManager(object):
         self._bg_sync_to_file()
         #TODO: wait for synchronization to suceed/fail??)
 
+    def _load_from_json(self, jsondata):
+        """Load fresh configuration data from jsondata
+
+        :param jsondata: String of jsondata
+        :return:
+        """
+        dumpdata = json.loads(jsondata)
+        tmpconfig = {}
+        for confarea in _config_areas:
+            if confarea not in dumpdata:
+                continue
+            tmpconfig[confarea] = {}
+            for element in dumpdata[confarea]:
+                newelement = copy.deepcopy(dumpdata[confarea][element])
+                for attribute in dumpdata[confarea][element]:
+                    if newelement[attribute] == '*REDACTED*':
+                        raise Exception(
+                            "Unable to restore from redacted backup")
+                    elif attribute == 'cryptpass':
+                        passparts = newelement[attribute].split('!')
+                        newelement[attribute] = tuple([base64.b64decode(x)
+                                                       for x in passparts])
+                    elif 'cryptvalue' in newelement[attribute]:
+                        bincrypt = newelement[attribute]['cryptvalue']
+                        bincrypt = tuple([base64.b64decode(x)
+                                          for x in bincrypt.split('!')])
+                        newelement[attribute]['cryptvalue'] = bincrypt
+                    elif attribute in ('nodes', '_expressionkeys'):
+                        # A group with nodes
+                        # delete it and defer until nodes are being added
+                        # which will implicitly fill this up
+                        # Or _expressionkeys attribute, which will similarly
+                        # be rebuilt
+                        del newelement[attribute]
+                tmpconfig[confarea][element] = newelement
+        # We made it through above section without an exception, go ahead and
+        # replace
+        # Start by erasing the dbm files if present
+        for confarea in _config_areas:
+            try:
+                os.unlink(os.path.join(self._cfgdir, confarea))
+            except OSError as e:
+                if e.errno == 2:
+                    pass
+        # Now we have to iterate through each fixed up element, using the
+        # set attribute to flesh out inheritence and expressions
+        for confarea in tmpconfig:
+            if confarea == 'nodes':
+                self.set_node_attributes(tmpconfig[confarea], True)
+            elif confarea == 'nodegroups':
+                self.set_group_attributes(tmpconfig[confarea], True)
+            elif confarea == 'users':
+                for user in tmpconfig[confarea]:
+                    uid = tmpconfig[confarea].get('id', None)
+                    displayname = tmpconfig[confarea].get('displayname', None)
+                    self.create_user(user, uid=uid, displayname=displayname)
+                    if 'cryptpass' in tmpconfig[confarea][user]:
+                        self._cfgstore['users'][user]['cryptpass'] = \
+                            tmpconfig[confarea][user]['cryptpass']
+                        _mark_dirtykey('users', user, self.tenant)
+        self._bg_sync_to_file()
+
     def _dump_to_json(self, redact=None):
         """Dump the configuration in json form to output
 
@@ -1403,6 +1465,16 @@ def _dump_keys(password):
                       sort_keys=True, indent=4, separators=(',', ': '))
 
 
+def restore_db_from_directory(location, password):
+    with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+        keydata = cfgfile.read()
+        json.loads(keydata)
+        _restore_keys(keydata, password)
+    with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
+        cfgdata = cfgfile.read()
+        ConfigManager(tenant=None)._load_from_json(cfgdata)
+
+
 def dump_db_to_directory(location, password, redact=None):
     with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
         cfgfile.write(_dump_keys(password))
@@ -1413,7 +1485,8 @@ def dump_db_to_directory(location, password, redact=None):
     try:
         for tenant in os.listdir(
                 os.path.join(ConfigManager._cfgdir, '/tenants/')):
-            with open(os.path.join(location, tenant + '.json'), 'w') as cfgfile:
+            with open(os.path.join(location, 'tenants', tenant,
+                                   'main.json'), 'w') as cfgfile:
                 cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
                     redact=redact))
                 cfgfile.write('\n')

From ae806e55b05305868a579d793dd968fffee9337f Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:08:28 -0500
Subject: [PATCH 43/64] Add a utility to frontend DB dump/restore

This exposes the library functions as a utility
---
 confluent_server/bin/confluentdbutil | 66 ++++++++++++++++++++++++++++
 confluent_server/confluent/main.py   | 20 +++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 confluent_server/bin/confluentdbutil

diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil
new file mode 100644
index 00000000..828f06da
--- /dev/null
+++ b/confluent_server/bin/confluentdbutil
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import optparse
+import sys
+import os
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    # if installed into system path, do not muck with things
+    sys.path.append(path)
+import confluent.config.configmanager as cfm
+import confluent.config.conf as conf
+import confluent.main as main
+
+argparser = optparse.OptionParser(usage="Usage: %prog [options] [dump|restore] [path]")
+argparser.add_option('-p', '--password',
+                     help='Password to use to protect/unlock a protected dump')
+argparser.add_option('-r', '--redact', action='store_true',
+                     help='Redact potentially sensitive data rather than store')
+argparser.add_option('-u', '--unprotected', action='store_true',
+                      help='Specify that no password should be used to protect'
+                           ' the key information.  Fields will be encrypted, '
+                           'but keys.json will contain unencrypted decryption'
+                           ' keys that may be used to read the dump')
+(options, args) = argparser.parse_args()
+if len(args) != 2 or args[0] not in ('dump', 'restore'):
+    argparser.print_help()
+    sys.exit(1)
+dumpdir = args[1]
+
+
+if args[0] == 'restore':
+    pid = main.is_running()
+    if pid is not None:
+        print("Confluent is running, must shut down to restore db")
+        sys.exit(1)
+    cfm.restore_db_from_directory(dumpdir, options.password)
+elif args[0] == 'dump':
+    if options.password is None and not options.unprotected:
+        print("Must indicate a password to protect or -u to opt opt of "
+              "secure value protection")
+        sys.exit(1)
+    os.umask(077)
+    main._initsecurity(conf.get_config())
+    if not os.path.exists(dumpdir):
+        os.makedirs(dumpdir)
+    cfm.dump_db_to_directory(dumpdir, options.password)
+
+
+
diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py
index 6e4ca9d6..2c5ce1d5 100644
--- a/confluent_server/confluent/main.py
+++ b/confluent_server/confluent/main.py
@@ -89,6 +89,26 @@ def _updatepidfile():
     pidfile.close()
 
 
+def is_running():
+    # Utility function for utilities to check if confluent is running
+    try:
+        pidfile = open('/var/run/confluent/pid', 'r+')
+        fcntl.flock(pidfile, fcntl.LOCK_SH)
+        pid = pidfile.read()
+        if pid != '':
+            try:
+                os.kill(int(pid), 0)
+                return pid
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        fcntl.flock(pidfile, fcntl.LOCK_UN)
+        pidfile.close()
+    except IOError:
+        pass
+    return None
+
+
 def _checkpidfile():
     try:
         pidfile = open('/var/run/confluent/pid', 'r+')

From 9bd0b7af9d741edef40e0df3c50ecdaf5d397aa8 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:10:01 -0500
Subject: [PATCH 44/64] Make confluentdbutil executable

---
 confluent_server/bin/confluentdbutil | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 confluent_server/bin/confluentdbutil

diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil
old mode 100644
new mode 100755

From 04781e0ecec696a8b69997784c976458c2075df8 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:12:49 -0500
Subject: [PATCH 45/64] Actually hook up the redact feature

---
 confluent_server/bin/confluentdbutil | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil
index 828f06da..e5acb419 100755
--- a/confluent_server/bin/confluentdbutil
+++ b/confluent_server/bin/confluentdbutil
@@ -28,7 +28,8 @@ import confluent.config.configmanager as cfm
 import confluent.config.conf as conf
 import confluent.main as main
 
-argparser = optparse.OptionParser(usage="Usage: %prog [options] [dump|restore] [path]")
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] [dump|restore] [path]")
 argparser.add_option('-p', '--password',
                      help='Password to use to protect/unlock a protected dump')
 argparser.add_option('-r', '--redact', action='store_true',
@@ -52,15 +53,15 @@ if args[0] == 'restore':
         sys.exit(1)
     cfm.restore_db_from_directory(dumpdir, options.password)
 elif args[0] == 'dump':
-    if options.password is None and not options.unprotected:
+    if options.password is None and not (options.unprotected or options.redact):
         print("Must indicate a password to protect or -u to opt opt of "
-              "secure value protection")
+              "secure value protection or -r to skip all protected data")
         sys.exit(1)
     os.umask(077)
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)
-    cfm.dump_db_to_directory(dumpdir, options.password)
+    cfm.dump_db_to_directory(dumpdir, options.password, options.redact)
 
 
 

From 5395f97a214ac9f3284d7756de72c931054dcc69 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:13:49 -0500
Subject: [PATCH 46/64] Do not write out keys when redacting

It's silly to store keys when redact is requested
---
 confluent_server/confluent/config/configmanager.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 7bf757be..2876b9d0 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1476,9 +1476,10 @@ def restore_db_from_directory(location, password):
 
 
 def dump_db_to_directory(location, password, redact=None):
-    with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
-        cfgfile.write(_dump_keys(password))
-        cfgfile.write('\n')
+    if not redact:
+        with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
+            cfgfile.write(_dump_keys(password))
+            cfgfile.write('\n')
     with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
         cfgfile.write('\n')

From 446d2270c9a90daccb480a0d13ccb01d983d84e0 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:27:49 -0500
Subject: [PATCH 47/64] Give a friendlier message on restore of redact DB

A redacted dump will not have a keys.json file, which
is natural.  Replace 'file not found' with a message
indicating the possibility of a redacted dump.
---
 confluent_server/confluent/config/configmanager.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 2876b9d0..10805ea7 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1466,10 +1466,15 @@ def _dump_keys(password):
 
 
 def restore_db_from_directory(location, password):
-    with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
-        keydata = cfgfile.read()
-        json.loads(keydata)
-        _restore_keys(keydata, password)
+    try:
+        with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+            keydata = cfgfile.read()
+            json.loads(keydata)
+            _restore_keys(keydata, password)
+    except IOError as e:
+        if e.errno == 2:
+            raise Exception("Cannot restore without keys, this may be a "
+                            "redacted dump")
     with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
         cfgdata = cfgfile.read()
         ConfigManager(tenant=None)._load_from_json(cfgdata)

From 6ad383c6adb37657b89c4213faf87f6128da6e80 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 30 Jan 2017 16:38:43 -0500
Subject: [PATCH 48/64] Assure deterministic enumeration of areas

nodegroup must be restored first.
---
 confluent_server/confluent/config/configmanager.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 10805ea7..4a36a97d 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1246,7 +1246,9 @@ class ConfigManager(object):
                     pass
         # Now we have to iterate through each fixed up element, using the
         # set attribute to flesh out inheritence and expressions
-        for confarea in tmpconfig:
+        for confarea in _config_areas:
+            if confarea not in tmpconfig:
+                continue
             if confarea == 'nodes':
                 self.set_node_attributes(tmpconfig[confarea], True)
             elif confarea == 'nodegroups':

From ecc6bcf96c0d8cdf453e2fecf86765f9cd2173c7 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 31 Jan 2017 15:38:54 -0500
Subject: [PATCH 49/64] Add the util to the setup.py

Previously packaging was missing the new utility
---
 confluent_server/setup.py.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl
index 9bd1d2ac..fdbbee84 100644
--- a/confluent_server/setup.py.tmpl
+++ b/confluent_server/setup.py.tmpl
@@ -5,7 +5,7 @@ setup(
     name='confluent_server',
     version='#VERSION#',
     author='Jarrod Johnson',
-    author_email='jbjohnso@us.ibm.com',
+    author_email='jjohnson2@lenovo.com',
     url='http://xcat.sf.net/',
     description='confluent systems management server',
     packages=['confluent', 'confluent/config', 'confluent/interface',
@@ -14,7 +14,7 @@ setup(
               'confluent/plugins/configuration/'],
     install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet',
                       'pyghmi>=0.6.5'],
-    scripts=['bin/confluent'],
+    scripts=['bin/confluent', 'bin/confluentdbutil'],
     data_files=[('/etc/init.d', ['sysvinit/confluent']),
                 ('/usr/lib/systemd/system', ['systemd/confluent.service']),
 		('/opt/confluent/lib/python/confluent/plugins/console/', [])],

From 9e593f05546d5520a6691fd822e7ee57a90e17ce Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 2 Feb 2017 17:00:09 -0500
Subject: [PATCH 50/64] Clean up nodepower

nodepower boot and reset output was misleading.
---
 confluent_client/bin/nodepower                   |  1 +
 confluent_client/confluent/client.py             | 11 ++++++++++-
 confluent_server/confluent/messages.py           |  5 +++++
 .../confluent/plugins/hardwaremanagement/ipmi.py | 16 ++++++++++++++--
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/confluent_client/bin/nodepower b/confluent_client/bin/nodepower
index 6711aea0..5dd0b007 100755
--- a/confluent_client/bin/nodepower
+++ b/confluent_client/bin/nodepower
@@ -45,5 +45,6 @@ if len(sys.argv) > 2:
 
 session = client.Command()
 exitcode = 0
+session.add_precede_key('oldstate')
 sys.exit(
     session.simple_noderange_command(noderange, '/power/state', setstate))
\ No newline at end of file
diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py
index 38ff2bbf..aa30e6a1 100644
--- a/confluent_client/confluent/client.py
+++ b/confluent_client/confluent/client.py
@@ -44,6 +44,7 @@ def _parseserver(string):
 class Command(object):
 
     def __init__(self, server=None):
+        self._prevkeyname = None
         self.connection = None
         if server is None:
             if 'CONFLUENT_HOST' in os.environ:
@@ -74,6 +75,9 @@ class Command(object):
         if authdata['authpassed'] == 1:
             self.authenticated = True
 
+    def add_precede_key(self, keyname):
+        self._prevkeyname = keyname
+
     def handle_results(self, ikey, rc, res):
         if 'error' in res:
             sys.stderr.write('Error: {0}\n'.format(res['error']))
@@ -93,7 +97,12 @@ class Command(object):
                 else:
                     rc |= 1
             elif ikey in res[node]:
-                print('{0}: {1}'.format(node, res[node][ikey]['value']))
+                if self._prevkeyname and self._prevkeyname in res[node]:
+                    print('{0}: {2}->{1}'.format(
+                        node, res[node][ikey]['value'],
+                        res[node][self._prevkeyname]['value']))
+                else:
+                    print('{0}: {1}'.format(node, res[node][ikey]['value']))
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
diff --git a/confluent_server/confluent/messages.py b/confluent_server/confluent/messages.py
index 4f43abe1..fc4a7df2 100644
--- a/confluent_server/confluent/messages.py
+++ b/confluent_server/confluent/messages.py
@@ -797,6 +797,11 @@ class PowerState(ConfluentChoiceMessage):
     ])
     keyname = 'state'
 
+    def __init__(self, node, state, oldstate=None):
+        super(PowerState, self).__init__(node, state)
+        if oldstate is not None:
+            self.kvpairs[node]['oldstate'] = {'value': oldstate}
+
 
 class BMCReset(ConfluentChoiceMessage):
     valid_values = set([
diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
index d929b2df..ce92b869 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py
@@ -795,10 +795,22 @@ class IpmiHandler(object):
             return
         elif 'update' == self.op:
             powerstate = self.inputdata.powerstate(self.node)
+            oldpower = None
+            if powerstate == 'boot':
+                oldpower = self.ipmicmd.get_power()
+                if 'powerstate' in oldpower:
+                    oldpower = oldpower['powerstate']
             self.ipmicmd.set_power(powerstate, wait=30)
-            power = self.ipmicmd.get_power()
+            if powerstate == 'boot' and oldpower == 'on':
+                power = {'powerstate': 'reset'}
+            else:
+                power = self.ipmicmd.get_power()
+                if powerstate == 'reset' and power['powerstate'] == 'on':
+                    power['powerstate'] = 'reset'
+
             self.output.put(msg.PowerState(node=self.node,
-                                           state=power['powerstate']))
+                                           state=power['powerstate'],
+                                           oldstate=oldpower))
             return
 
     def handle_reset(self):

From b39ae429557001a8708df6e40ecfe7ead2e44202 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 10 Feb 2017 16:41:47 -0500
Subject: [PATCH 51/64] Add debug of broken expression to nodelist

If an expression is invalid, have nodelist give the error data
allowing the user to see and potentially take action.
---
 confluent_client/bin/nodelist | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist
index 50245628..6cff5dcd 100755
--- a/confluent_client/bin/nodelist
+++ b/confluent_client/bin/nodelist
@@ -76,7 +76,11 @@ if len(args) > 1:
                             attrout = '{0}: {1}: ********'.format(node, attr)
                         else:
                             attrout = '{0}: {1}:'.format(node, attr)
-                    if options.blame:
+                    elif 'broken' in currattr:
+                        attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                                  '{2}'.format(node, attr,
+                                               currattr['broken'])
+                    if options.blame or 'broken' in currattr:
                         blamedata = []
                         if 'inheritedfrom' in currattr:
                             blamedata.append('inherited from group {0}'.format(

From f3cfe4ee2675b97cf31b85954704a221310ccf9a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 15 Feb 2017 10:58:08 -0500
Subject: [PATCH 52/64] Change strategy for stale data protection

Rather than assuming a global state, have the generator react to
GeneratorExit and clean itself up.
---
 confluent_client/confluent/client.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py
index 07a85423..b83050e1 100644
--- a/confluent_client/confluent/client.py
+++ b/confluent_client/confluent/client.py
@@ -27,8 +27,6 @@ import confluent.tlvdata as tlvdata
 SO_PASSCRED = 16
 
 
-inflight = False
-
 def _parseserver(string):
     if ']:' in string:
         server, port = string[1:].split(']:')
@@ -228,23 +226,19 @@ def send_request(operation, path, server, parameters=None):
     :param server: The socket to send data over
     :param parameters:  Parameters if any to send along with the request
     """
-    global inflight
-    if inflight:
-        # calling code failed to complete a transaction, flush and discard
-        # their unused data
-        result = tlvdata.recv(server)
-        while '_requestdone' not in result:
-            result = tlvdata.recv(server)
-    inflight = True
     payload = {'operation': operation, 'path': path}
     if parameters is not None:
         payload['parameters'] = parameters
     tlvdata.send(server, payload)
     result = tlvdata.recv(server)
     while '_requestdone' not in result:
-        yield result
+        try:
+            yield result
+        except GeneratorExit:
+            while '_requestdone' not in result:
+                result = tlvdata.recv(server)
+            raise
         result = tlvdata.recv(server)
-    inflight = False
 
 
 

From bbfed443fc47bf48d7b6425bbde76c9830bf75a5 Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Fri, 17 Feb 2017 10:44:25 -0500
Subject: [PATCH 53/64] Adding nodeattrib to list and update attributes

---
 confluent_client/bin/nodeattrib | 133 ++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 confluent_client/bin/nodeattrib

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
new file mode 100644
index 00000000..93b72fa7
--- /dev/null
+++ b/confluent_client/bin/nodeattrib
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'alin37'
+
+import optparse
+import os
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+
+def attrrequested(attr, attrlist, seenattributes):
+    for candidate in attrlist:
+        truename = candidate
+        if candidate.startswith('hm'):
+            candidate = candidate.replace('hm', 'hardwaremanagement', 1)
+        if candidate == attr:
+            seenattributes.add(truename)
+            return True
+        elif '.' not in candidate and attr.startswith(candidate + '.'):
+            seenattributes.add(truename)
+            return True
+    return False
+argparser = optparse.OptionParser(
+    usage='''Usage: %prog [options] noderange [list of attributes]\n  \
+                  %prog [options] noderange attribute1=value1,attribute2=value,...   ''')
+argparser.add_option('-b', '--blame', action='store_true',
+                     help='Show information about how attributes inherited')
+(options, args) = argparser.parse_args()
+
+showtype = 'all'
+requestargs=None
+try:
+    noderange = args[0]
+    nodelist = '/noderange/{0}/nodes/'.format(noderange)
+except IndexError:
+    nodelist = '/nodes/'
+session = client.Command()
+exitcode = 0
+
+if len(args) > 1:
+    if args[1]  == 'all':
+        showtype = 'all'
+    elif args[1] == 'current':
+        showtype = 'current'
+    elif "=" in args[1]:
+        try:
+            if len(args[1:]) > 1:
+                for val in args[1:].split(','):
+                    val = val.split('=')
+                    exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0])
+            else:
+                val=args[1].split('=')
+                exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0])
+        except:
+            sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:])))
+            exitcode = 1
+        sys.exit(exitcode)
+    else:
+        requestargs = args[1:]
+
+if len(args) > 0:
+    seenattributes = set([])
+    for res in session.read('/noderange/{0}/attributes/{1}'.format(noderange,showtype)):
+        if 'error' in res:
+            print "found error"
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+            continue
+        for node in res['databynode']:
+            for attr in res['databynode'][node]:
+                seenattributes.add(attr)
+                currattr = res['databynode'][node][attr]
+                if requestargs is None or attrrequested(attr, args[1:], seenattributes):
+                    if 'value' in currattr:
+                        if currattr['value'] is not None:
+                           attrout = '{0}: {1}: {2}'.format(
+                                node, attr, currattr['value'])
+                        else:
+                            attrout = '{0}: {1}:'.format(node, attr)
+                    elif 'isset' in currattr:
+                        if currattr['isset']:
+                            attrout = '{0}: {1}: ********'.format(node, attr)
+                        else:
+                            attrout = '{0}: {1}:'.format(node, attr)
+                    if options.blame:
+                        blamedata = []
+                        if 'inheritedfrom' in currattr:
+                            blamedata.append('inherited from group {0}'.format(
+                                currattr['inheritedfrom']
+                            ))
+                        if 'expression' in currattr:
+                            blamedata.append(
+                                'derived from expression "{0}"'.format(
+                                    currattr['expression']))
+                        if blamedata:
+                            attrout += ' (' + ', '.join(blamedata) + ')'
+                    print attrout
+
+    if not exitcode:
+        if requestargs:
+            for attr in args[1:]:
+                if attr not in seenattributes:
+                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
+                    exitcode = 1
+else:
+    for res in session.read(nodelist):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print res['item']['href'].replace('/', '')
+sys.exit(exitcode)
\ No newline at end of file

From 60a1ba77b7fba3375ddf11e4c34e633435776420 Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Fri, 17 Feb 2017 10:46:57 -0500
Subject: [PATCH 54/64] Setting nodeattrib default to show current

---
 confluent_client/bin/nodeattrib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index 93b72fa7..8874a472 100644
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -48,7 +48,7 @@ argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
 (options, args) = argparser.parse_args()
 
-showtype = 'all'
+showtype = 'current'
 requestargs=None
 try:
     noderange = args[0]

From b714cfdf0f3d9baf55bd7f5ad516a2967a064212 Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Tue, 28 Feb 2017 13:27:22 -0500
Subject: [PATCH 55/64] Adding nodeattrib to change and update node attributes

---
 confluent_client/bin/nodeattrib          | 58 +++++++++++-------
 confluent_client/doc/man/nodeattrib.ronn | 76 ++++++++++++++++++++++++
 2 files changed, 114 insertions(+), 20 deletions(-)
 create mode 100644 confluent_client/doc/man/nodeattrib.ronn

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index 8874a472..2ae05aa1 100644
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -28,7 +28,6 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
 def attrrequested(attr, attrlist, seenattributes):
     for candidate in attrlist:
         truename = candidate
@@ -42,10 +41,13 @@ def attrrequested(attr, attrlist, seenattributes):
             return True
     return False
 argparser = optparse.OptionParser(
-    usage='''Usage: %prog [options] noderange [list of attributes]\n  \
-                  %prog [options] noderange attribute1=value1,attribute2=value,...   ''')
+    usage='''\n       %prog [options] noderange [list of attributes]  \
+             \n       %prog [options] noderange attribute1=value1,attribute2=value,...
+             \n ''')
 argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
+argparser.add_option('-c', '--clear', action='store_true',
+                     help='Clear variables')
 (options, args) = argparser.parse_args()
 
 showtype = 'current'
@@ -58,27 +60,43 @@ except IndexError:
 session = client.Command()
 exitcode = 0
 
+
+#Sets attributes
 if len(args) > 1:
-    if args[1]  == 'all':
-        showtype = 'all'
-    elif args[1] == 'current':
-        showtype = 'current'
-    elif "=" in args[1]:
-        try:
-            if len(args[1:]) > 1:
-                for val in args[1:].split(','):
-                    val = val.split('=')
-                    exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0])
-            else:
-                val=args[1].split('=')
-                exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0])
-        except:
-            sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:])))
-            exitcode = 1
+    #clears attribute
+    if options.clear:
+        targpath = '/noderange/{0}/attributes/all'.format(noderange)
+        keydata = {}
+        for attrib in args[1:]:
+            keydata[attrib] = None
+        for res in session.update(targpath, keydata):
+            if 'error' in res:
+                if 'errorcode' in res:
+                    exitcode = res['errorcode']
+                sys.stderr.write('Error: ' + res['error'] + '\n')
         sys.exit(exitcode)
     else:
-        requestargs = args[1:]
+        if args[1]  == 'all':
+            showtype = 'all'
+        elif args[1] == 'current':
+            showtype = 'current'
+        elif "=" in args[1]:
+            try:
+                if len(args[1:]) > 1:
+                    for val in args[1:]:
+                        val = val.split('=')
+                        exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0])
+                else:
+                    val=args[1].split('=')
+                    exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0])
+            except:
+                sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:])))
+                exitcode = 1
+            sys.exit(exitcode)
+        else:
+            requestargs = args[1:]
 
+# Lists all attributes
 if len(args) > 0:
     seenattributes = set([])
     for res in session.read('/noderange/{0}/attributes/{1}'.format(noderange,showtype)):
diff --git a/confluent_client/doc/man/nodeattrib.ronn b/confluent_client/doc/man/nodeattrib.ronn
new file mode 100644
index 00000000..63ecf145
--- /dev/null
+++ b/confluent_client/doc/man/nodeattrib.ronn
@@ -0,0 +1,76 @@
+nodeattrib(1) -- List or change confluent nodes attributes
+=========================================================
+
+## SYNOPSIS
+
+`nodeattrib` `noderange` [ current | all ]
+`nodeattrib` `noderange` [-b] [<nodeattribute>...]
+`nodeattrib` `noderange` [<nodeattribute1=value1> <nodeattribute2=value2> ...]
+`nodeattrib` `noderange` [-c] [<nodeattribute1> <nodeattribute2=value2> ...]
+
+## DESCRIPTION
+
+**nodeattrib** queries the confluent server to get information about nodes.  In
+the simplest form, it simply takes the given noderange(5) and lists the
+matching nodes, one line at a time.
+
+If a list of node attribute names are given, the value of those are also
+displayed.  If `-b` is specified, it will also display information on
+how inherited and expression based attributes are defined.  There is more
+information on node attributes in nodeattributes(5) man page.
+If `-c` is specified, this will set the nodeattribute to a null valid.
+This is different from setting the value to an empty string.
+
+## OPTIONS
+
+* `-b`, `--blame`:
+  Annotate inherited and expression based attributes to show their base value.
+* `-c`, `--clear`:
+  Clear given nodeattributes since '' is not the same as empty
+
+## EXAMPLES
+* Listing matching nodes of a simple noderange:
+    `# nodeattrib n1-n2`
+    `n1`: console.method: ipmi
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method: ipmi
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+* Getting an attribute of nodes matching a noderange:
+    `# nodeattrib n1,n2 hardwaremanagement.manager`
+    `n1: hardwaremanagement.manager: 172.30.3.1`
+    `n2: hardwaremanagement.manager: 172.30.3.2`
+
+* Getting a group of attributes while determining what group defines them:
+    `# nodeattrib n1,n2 hardwaremanagement --blame`
+    `n1: hardwaremanagement.manager: 172.30.3.1`
+    `n1: hardwaremanagement.method: ipmi (inherited from group everything)`
+    `n1: hardwaremanagement.switch: r8e1`
+    `n1: hardwaremanagement.switchport: 14`
+    `n2: hardwaremanagement.manager: 172.30.3.2`
+    `n2: hardwaremanagement.method: ipmi (inherited from group everything)`
+    `n2: hardwaremanagement.switch: r8e1`
+    `n2: hardwaremanagement.switchport: 2`
+
+  * Listing matching nodes of a simple noderange that are set:
+    `# nodeattrib n1-n2 current`
+    `n1`: console.method: ipmi
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method: ipmi
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+  * Change attribute on nodes of a simple noderange:
+    `# nodeattrib n1-n2 console.method=serial`
+    `n1`: console.method: serial
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method: serial
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+  * Clear attribute on nodes of a simple noderange:
+    `# nodeattrib n1-n2 -c console.method`
+    `n1`: console.method:
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method:
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+

From 2d8004000d032c7c004acc59d613e72dddd96975 Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Fri, 17 Feb 2017 10:44:25 -0500
Subject: [PATCH 56/64] Adding nodeattrib to list and update attributes

---
 confluent_client/bin/nodeattrib | 133 ++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 confluent_client/bin/nodeattrib

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
new file mode 100644
index 00000000..93b72fa7
--- /dev/null
+++ b/confluent_client/bin/nodeattrib
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'alin37'
+
+import optparse
+import os
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+
+def attrrequested(attr, attrlist, seenattributes):
+    for candidate in attrlist:
+        truename = candidate
+        if candidate.startswith('hm'):
+            candidate = candidate.replace('hm', 'hardwaremanagement', 1)
+        if candidate == attr:
+            seenattributes.add(truename)
+            return True
+        elif '.' not in candidate and attr.startswith(candidate + '.'):
+            seenattributes.add(truename)
+            return True
+    return False
+argparser = optparse.OptionParser(
+    usage='''Usage: %prog [options] noderange [list of attributes]\n  \
+                  %prog [options] noderange attribute1=value1,attribute2=value,...   ''')
+argparser.add_option('-b', '--blame', action='store_true',
+                     help='Show information about how attributes inherited')
+(options, args) = argparser.parse_args()
+
+showtype = 'all'
+requestargs=None
+try:
+    noderange = args[0]
+    nodelist = '/noderange/{0}/nodes/'.format(noderange)
+except IndexError:
+    nodelist = '/nodes/'
+session = client.Command()
+exitcode = 0
+
+if len(args) > 1:
+    if args[1]  == 'all':
+        showtype = 'all'
+    elif args[1] == 'current':
+        showtype = 'current'
+    elif "=" in args[1]:
+        try:
+            if len(args[1:]) > 1:
+                for val in args[1:].split(','):
+                    val = val.split('=')
+                    exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0])
+            else:
+                val=args[1].split('=')
+                exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0])
+        except:
+            sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:])))
+            exitcode = 1
+        sys.exit(exitcode)
+    else:
+        requestargs = args[1:]
+
+if len(args) > 0:
+    seenattributes = set([])
+    for res in session.read('/noderange/{0}/attributes/{1}'.format(noderange,showtype)):
+        if 'error' in res:
+            print "found error"
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+            continue
+        for node in res['databynode']:
+            for attr in res['databynode'][node]:
+                seenattributes.add(attr)
+                currattr = res['databynode'][node][attr]
+                if requestargs is None or attrrequested(attr, args[1:], seenattributes):
+                    if 'value' in currattr:
+                        if currattr['value'] is not None:
+                           attrout = '{0}: {1}: {2}'.format(
+                                node, attr, currattr['value'])
+                        else:
+                            attrout = '{0}: {1}:'.format(node, attr)
+                    elif 'isset' in currattr:
+                        if currattr['isset']:
+                            attrout = '{0}: {1}: ********'.format(node, attr)
+                        else:
+                            attrout = '{0}: {1}:'.format(node, attr)
+                    if options.blame:
+                        blamedata = []
+                        if 'inheritedfrom' in currattr:
+                            blamedata.append('inherited from group {0}'.format(
+                                currattr['inheritedfrom']
+                            ))
+                        if 'expression' in currattr:
+                            blamedata.append(
+                                'derived from expression "{0}"'.format(
+                                    currattr['expression']))
+                        if blamedata:
+                            attrout += ' (' + ', '.join(blamedata) + ')'
+                    print attrout
+
+    if not exitcode:
+        if requestargs:
+            for attr in args[1:]:
+                if attr not in seenattributes:
+                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
+                    exitcode = 1
+else:
+    for res in session.read(nodelist):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print res['item']['href'].replace('/', '')
+sys.exit(exitcode)
\ No newline at end of file

From 1b44d2d7815ef6ace2508ef333e3dab3ead4896e Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Fri, 17 Feb 2017 10:46:57 -0500
Subject: [PATCH 57/64] Setting nodeattrib default to show current

---
 confluent_client/bin/nodeattrib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index 93b72fa7..8874a472 100644
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -48,7 +48,7 @@ argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
 (options, args) = argparser.parse_args()
 
-showtype = 'all'
+showtype = 'current'
 requestargs=None
 try:
     noderange = args[0]

From c78b7fa14609fe3bc7751dc81d85fa0b00af77ad Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Tue, 28 Feb 2017 13:27:22 -0500
Subject: [PATCH 58/64] Adding nodeattrib to change and update node attributes

---
 confluent_client/bin/nodeattrib          | 58 +++++++++++-------
 confluent_client/doc/man/nodeattrib.ronn | 76 ++++++++++++++++++++++++
 2 files changed, 114 insertions(+), 20 deletions(-)
 create mode 100644 confluent_client/doc/man/nodeattrib.ronn

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index 8874a472..2ae05aa1 100644
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -28,7 +28,6 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
 def attrrequested(attr, attrlist, seenattributes):
     for candidate in attrlist:
         truename = candidate
@@ -42,10 +41,13 @@ def attrrequested(attr, attrlist, seenattributes):
             return True
     return False
 argparser = optparse.OptionParser(
-    usage='''Usage: %prog [options] noderange [list of attributes]\n  \
-                  %prog [options] noderange attribute1=value1,attribute2=value,...   ''')
+    usage='''\n       %prog [options] noderange [list of attributes]  \
+             \n       %prog [options] noderange attribute1=value1,attribute2=value,...
+             \n ''')
 argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
+argparser.add_option('-c', '--clear', action='store_true',
+                     help='Clear variables')
 (options, args) = argparser.parse_args()
 
 showtype = 'current'
@@ -58,27 +60,43 @@ except IndexError:
 session = client.Command()
 exitcode = 0
 
+
+#Sets attributes
 if len(args) > 1:
-    if args[1]  == 'all':
-        showtype = 'all'
-    elif args[1] == 'current':
-        showtype = 'current'
-    elif "=" in args[1]:
-        try:
-            if len(args[1:]) > 1:
-                for val in args[1:].split(','):
-                    val = val.split('=')
-                    exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0])
-            else:
-                val=args[1].split('=')
-                exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0])
-        except:
-            sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:])))
-            exitcode = 1
+    #clears attribute
+    if options.clear:
+        targpath = '/noderange/{0}/attributes/all'.format(noderange)
+        keydata = {}
+        for attrib in args[1:]:
+            keydata[attrib] = None
+        for res in session.update(targpath, keydata):
+            if 'error' in res:
+                if 'errorcode' in res:
+                    exitcode = res['errorcode']
+                sys.stderr.write('Error: ' + res['error'] + '\n')
         sys.exit(exitcode)
     else:
-        requestargs = args[1:]
+        if args[1]  == 'all':
+            showtype = 'all'
+        elif args[1] == 'current':
+            showtype = 'current'
+        elif "=" in args[1]:
+            try:
+                if len(args[1:]) > 1:
+                    for val in args[1:]:
+                        val = val.split('=')
+                        exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0])
+                else:
+                    val=args[1].split('=')
+                    exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0])
+            except:
+                sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:])))
+                exitcode = 1
+            sys.exit(exitcode)
+        else:
+            requestargs = args[1:]
 
+# Lists all attributes
 if len(args) > 0:
     seenattributes = set([])
     for res in session.read('/noderange/{0}/attributes/{1}'.format(noderange,showtype)):
diff --git a/confluent_client/doc/man/nodeattrib.ronn b/confluent_client/doc/man/nodeattrib.ronn
new file mode 100644
index 00000000..63ecf145
--- /dev/null
+++ b/confluent_client/doc/man/nodeattrib.ronn
@@ -0,0 +1,76 @@
+nodeattrib(1) -- List or change confluent nodes attributes
+=========================================================
+
+## SYNOPSIS
+
+`nodeattrib` `noderange` [ current | all ]
+`nodeattrib` `noderange` [-b] [<nodeattribute>...]
+`nodeattrib` `noderange` [<nodeattribute1=value1> <nodeattribute2=value2> ...]
+`nodeattrib` `noderange` [-c] [<nodeattribute1> <nodeattribute2=value2> ...]
+
+## DESCRIPTION
+
+**nodeattrib** queries the confluent server to get information about nodes.  In
+the simplest form, it simply takes the given noderange(5) and lists the
+matching nodes, one line at a time.
+
+If a list of node attribute names are given, the value of those are also
+displayed.  If `-b` is specified, it will also display information on
+how inherited and expression based attributes are defined.  There is more
+information on node attributes in nodeattributes(5) man page.
+If `-c` is specified, this will set the nodeattribute to a null valid.
+This is different from setting the value to an empty string.
+
+## OPTIONS
+
+* `-b`, `--blame`:
+  Annotate inherited and expression based attributes to show their base value.
+* `-c`, `--clear`:
+  Clear given nodeattributes since '' is not the same as empty
+
+## EXAMPLES
+* Listing matching nodes of a simple noderange:
+    `# nodeattrib n1-n2`
+    `n1`: console.method: ipmi
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method: ipmi
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+* Getting an attribute of nodes matching a noderange:
+    `# nodeattrib n1,n2 hardwaremanagement.manager`
+    `n1: hardwaremanagement.manager: 172.30.3.1`
+    `n2: hardwaremanagement.manager: 172.30.3.2`
+
+* Getting a group of attributes while determining what group defines them:
+    `# nodeattrib n1,n2 hardwaremanagement --blame`
+    `n1: hardwaremanagement.manager: 172.30.3.1`
+    `n1: hardwaremanagement.method: ipmi (inherited from group everything)`
+    `n1: hardwaremanagement.switch: r8e1`
+    `n1: hardwaremanagement.switchport: 14`
+    `n2: hardwaremanagement.manager: 172.30.3.2`
+    `n2: hardwaremanagement.method: ipmi (inherited from group everything)`
+    `n2: hardwaremanagement.switch: r8e1`
+    `n2: hardwaremanagement.switchport: 2`
+
+  * Listing matching nodes of a simple noderange that are set:
+    `# nodeattrib n1-n2 current`
+    `n1`: console.method: ipmi
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method: ipmi
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+  * Change attribute on nodes of a simple noderange:
+    `# nodeattrib n1-n2 console.method=serial`
+    `n1`: console.method: serial
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method: serial
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+  * Clear attribute on nodes of a simple noderange:
+    `# nodeattrib n1-n2 -c console.method`
+    `n1`: console.method:
+    `n1`: hardwaremanagement.manager: 172.30.3.1
+    `n2`: console.method:
+    `n2`: hardwaremanagement.manager: 172.30.3.2
+
+

From 16f2a2b1ebbd5c6b1b46943ccee57a04246df838 Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Tue, 28 Feb 2017 13:51:36 -0500
Subject: [PATCH 59/64] Updated for broken expressions

---
 confluent_client/bin/nodeattrib          | 6 +++++-
 confluent_client/doc/man/nodeattrib.ronn | 9 ++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index 2ae05aa1..e0120be4 100644
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -121,7 +121,11 @@ if len(args) > 0:
                             attrout = '{0}: {1}: ********'.format(node, attr)
                         else:
                             attrout = '{0}: {1}:'.format(node, attr)
-                    if options.blame:
+                    elif 'broken' in currattr:
+                        attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                                  '{2}'.format(node, attr,
+                                               currattr['broken'])
+                    if options.blame or 'broken' in currattr:
                         blamedata = []
                         if 'inheritedfrom' in currattr:
                             blamedata.append('inherited from group {0}'.format(
diff --git a/confluent_client/doc/man/nodeattrib.ronn b/confluent_client/doc/man/nodeattrib.ronn
index 63ecf145..dc330b0c 100644
--- a/confluent_client/doc/man/nodeattrib.ronn
+++ b/confluent_client/doc/man/nodeattrib.ronn
@@ -66,11 +66,10 @@ This is different from setting the value to an empty string.
     `n2`: console.method: serial
     `n2`: hardwaremanagement.manager: 172.30.3.2
 
-  * Clear attribute on nodes of a simple noderange:
+  * Clear attribute on nodes of a simple noderange, if you want to retain the variable set the attribute to "":
     `# nodeattrib n1-n2 -c console.method`
-    `n1`: console.method:
-    `n1`: hardwaremanagement.manager: 172.30.3.1
-    `n2`: console.method:
-    `n2`: hardwaremanagement.manager: 172.30.3.2
+    `# nodeattrib n1-n2 console.method`
+    Error: console.logging not a valid attribute
+
 
 

From 011ba663146e58205429f19bfb6cadf8f751da58 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 1 Mar 2017 10:24:23 -0500
Subject: [PATCH 60/64] Add a -u option for symmetry with rsetboot

If people want to alias xCAT commands, then this
is required for compatibility.
---
 confluent_client/bin/nodesetboot | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/confluent_client/bin/nodesetboot b/confluent_client/bin/nodesetboot
index bbe71d46..65f8aaeb 100755
--- a/confluent_client/bin/nodesetboot
+++ b/confluent_client/bin/nodesetboot
@@ -35,6 +35,9 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
                      default=False,
                      help='Request the boot device be persistent rather than '
                           'one time')
+argparser.add_option('-u', '--uefi', dest='uefi', action='store_true',
+                     default=True,
+                     help='Request UEFI style boot (rather than BIOS)')
 
 (options, args) = argparser.parse_args()
 

From 43b51eec20f607fe88a2649cfab388e7dcd72a3a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 1 Mar 2017 10:31:00 -0500
Subject: [PATCH 61/64] Be more friendly about shlex parsing errors

If user provides bad input, be more helpful and
less fatalistic.
---
 confluent_client/bin/confetty | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/confluent_client/bin/confetty b/confluent_client/bin/confetty
index 4b34490f..64764a5e 100755
--- a/confluent_client/bin/confetty
+++ b/confluent_client/bin/confetty
@@ -235,7 +235,11 @@ def rcompleter(text, state):
 
 
 def parse_command(command):
-    args = shlex.split(command, posix=True)
+    try:
+        args = shlex.split(command, posix=True)
+    except ValueError as ve:
+        print('Error: ' + ve.message)
+        return []
     return args
 
 

From fd64a2e68c2f1333371209d6ee830ff8d0a025c7 Mon Sep 17 00:00:00 2001
From: Allen Lin37 <alin37@Lenovo.com>
Date: Fri, 3 Mar 2017 13:10:29 -0500
Subject: [PATCH 62/64] Fix nodelist and nodeattrib to read attributes in
 groups

---
 confluent_client/bin/nodeattrib | 10 ++++++++++
 confluent_client/bin/nodelist   | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index e0120be4..a05915c6 100644
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -125,6 +125,16 @@ if len(args) > 0:
                         attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
                                   '{2}'.format(node, attr,
                                                currattr['broken'])
+                    elif isinstance(currattr, list) or isinstance(currattr, tuple):
+                        attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, currattr)))
+                    elif isinstance(currattr, dict):
+                        dictout = []
+                        for k,v in currattr.items:
+                            dictout.append("{0}={1}".format(k,v))
+                        attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, dictout)))
+                    else:
+                        print ("CODE ERROR" + repr(attr))
+
                     if options.blame or 'broken' in currattr:
                         blamedata = []
                         if 'inheritedfrom' in currattr:
diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist
index 6cff5dcd..92ea8320 100755
--- a/confluent_client/bin/nodelist
+++ b/confluent_client/bin/nodelist
@@ -80,6 +80,16 @@ if len(args) > 1:
                         attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
                                   '{2}'.format(node, attr,
                                                currattr['broken'])
+                    elif isinstance(currattr, list) or isinstance(currattr, tuple):
+                        attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, currattr)))
+                    elif isinstance(currattr, dict):
+                        dictout = []
+                        for k, v in currattr.items:
+                            dictout.append("{0}={1}".format(k, v))
+                        attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, dictout)))
+                    else:
+                        print ("CODE ERROR" + repr(attr))
+
                     if options.blame or 'broken' in currattr:
                         blamedata = []
                         if 'inheritedfrom' in currattr:

From 919dab9b5561e42471434c13404ceb8e85495a40 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Sun, 5 Mar 2017 08:28:35 -0500
Subject: [PATCH 63/64] Create auth multiprocessing pool on demand

Most of the time, we don't need this pool.  Create when needed,
and clean up after 30 seconds of inactivity.  This avoids a slow
shutdown that was due to core python hanging in help_finish_stuff,
and as a bonus means most of the time, one only sees one confluent
process, which has been a source of questions already.
---
 confluent_server/confluent/auth.py | 27 ++++++++++++++++-----------
 confluent_server/confluent/main.py |  1 -
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/confluent_server/confluent/auth.py b/confluent_server/confluent/auth.py
index 452cdc40..85fc1b14 100644
--- a/confluent_server/confluent/auth.py
+++ b/confluent_server/confluent/auth.py
@@ -37,6 +37,7 @@ _passcache = {}
 _passchecking = {}
 
 authworkers = None
+authcleaner = None
 
 
 class Credentials(object):
@@ -195,6 +196,13 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
     #such a beast could be passed into pyghmi as a way for pyghmi to
     #magically get offload of the crypto functions without having
     #to explicitly get into the eventlet tpool game
+    global authworkers
+    global authcleaner
+    if authworkers is None:
+        authworkers = multiprocessing.Pool(processes=1)
+    else:
+        authcleaner.cancel()
+    authcleaner = eventlet.spawn_after(30, _clean_authworkers)
     crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
     del _passchecking[(user, tenant)]
     eventlet.sleep(0.05)  # either way, we want to stall so that client can't
@@ -211,19 +219,16 @@ def _apply_pbkdf(passphrase, salt):
                       lambda p, s: hmac.new(p, s, hashlib.sha256).digest())
 
 
+def _clean_authworkers():
+    global authworkers
+    global authcleaner
+    authworkers = None
+    authcleaner = None
+
+
 def _do_pbkdf(passphrase, salt):
     # we must get it over to the authworkers pool or else get blocked in
     # compute.  However, we do want to wait for result, so we have
     # one of the exceedingly rare sort of circumstances where 'apply'
     # actually makes sense
-    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
-
-
-def init_auth():
-    # have a some auth workers available.  Keep them distinct from
-    # the general populace of workers to avoid unauthorized users
-    # starving out productive work
-    global authworkers
-    # for now we'll just have one auth worker and see if there is any
-    # demand for more.  I personally doubt it.
-    authworkers = multiprocessing.Pool(processes=1)
+    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
\ No newline at end of file
diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py
index 2c5ce1d5..cd26fba5 100644
--- a/confluent_server/confluent/main.py
+++ b/confluent_server/confluent/main.py
@@ -222,7 +222,6 @@ def run():
     _daemonize()
     if havefcntl:
         _updatepidfile()
-    auth.init_auth()
     signal.signal(signal.SIGINT, terminate)
     signal.signal(signal.SIGTERM, terminate)
     if dbgif:

From 236d889d5df197add0370e43b8673f2a6ef00168 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Sun, 5 Mar 2017 09:20:50 -0500
Subject: [PATCH 64/64] Fix error on 'isset' data

isset data is to be redacted.  This handles that
situation, and also provides a repr if all else fails.
---
 confluent_client/confluent/client.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py
index b83050e1..5d213003 100644
--- a/confluent_client/confluent/client.py
+++ b/confluent_client/confluent/client.py
@@ -97,12 +97,17 @@ class Command(object):
                 else:
                     rc |= 1
             elif ikey in res[node]:
+                if 'value' in res[node][ikey]:
+                    val = res[node][ikey]['value']
+                elif 'isset' in res[node][ikey]:
+                    val = '********' if res[node][ikey] else ''
+                else:
+                    val = repr(res[node][ikey])
                 if self._prevkeyname and self._prevkeyname in res[node]:
                     print('{0}: {2}->{1}'.format(
-                        node, res[node][ikey]['value'],
-                        res[node][self._prevkeyname]['value']))
+                        node, val, res[node][self._prevkeyname]['value']))
                 else:
-                    print('{0}: {1}'.format(node, res[node][ikey]['value']))
+                    print('{0}: {1}'.format(node, val))
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,