Merge branch 'master' into async

README.md

@@ -0,0 +1,30 @@
+# Confluent
+
+![Python 3](https://img.shields.io/badge/python-3-blue.svg) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/xcat2/confluent/blob/master/LICENSE)
+
+Confluent is a software package to handle essential bootstrap and operation of scale-out server configurations.
+It supports stateful and stateless deployments for various operating systems.
+
+Check [this page](https://hpc.lenovo.com/users/documentation/whatisconfluent.html
+) for a more detailed list of features.
+
+Confluent is the modern successor of [xCAT](https://github.com/xcat2/xcat-core).
+If you're coming from xCAT, check out [this comparison](https://hpc.lenovo.com/users/documentation/confluentvxcat.html).
+
+# Documentation
+
+Confluent documentation is hosted on hpc.lenovo.com: https://hpc.lenovo.com/users/documentation/
+
+# Download
+
+Get the latest version from: https://hpc.lenovo.com/users/downloads/
+
+Check release notes on: https://hpc.lenovo.com/users/news/
+
+# Open Source License
+
+Confluent is made available under the Apache 2.0 license: https://opensource.org/license/apache-2-0
+
+# Developers
+
+Want to help? Submit a [Pull Request](https://github.com/xcat2/confluent/pulls).

confluent_client/confluent_client.spec.tmpl

@@ -1,12 +1,16 @@
 %define name confluent_client
 %define version #VERSION#
+%define fversion %{lua:
+sv, _ = string.gsub("#VERSION#", "[~+]", "-")
+print(sv)
+}
 %define release 1
 
 Summary: Client libraries and utilities for confluent
 Name: %{name}
 Version: %{version}
 Release: %{release}
-Source0: %{name}-%{version}.tar.gz
+Source0: %{name}-%{fversion}.tar.gz
 License: Apache2
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
@@ -21,7 +25,7 @@ This package enables python development and command line access to
 a confluent server.
 
 %prep
-%setup -n %{name}-%{version} -n %{name}-%{version}
+%setup -n %{name}-%{fversion}
 
 %build
 %if "%{dist}" == ".el7"

confluent_osdeploy/buildrpm

@@ -1,7 +1,11 @@
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec
 cd ..

confluent_osdeploy/buildrpm-aarch64

@@ -2,7 +2,11 @@ cd $(dirname $0)
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy-aarch64.spec.tmpl > confluent_osdeploy-aarch64.spec
 cd ..

confluent_server/builddeb

@@ -17,7 +17,9 @@ cd /tmp/confluent/$PKGNAME
 if [ -x ./makeman ]; then
     ./makeman
 fi
-./makesetup
+sed -e 's/~/./' ./makesetup > ./makesetup.deb
+chmod +x ./makesetup.deb
+./makesetup.deb
 VERSION=`cat VERSION`
 cat > setup.cfg << EOF
 [install]
@@ -74,7 +76,7 @@ else
     rm -rf $PKGNAME.egg-info dist setup.py
     rm -rf $(find deb_dist -mindepth 1 -maxdepth 1 -type d)
     if [ ! -z "$1" ]; then
-        mv deb_dist/* $1/
+        mv deb_dist/*.deb $1/
     fi
 fi
 exit 0

confluent_server/confluent/core.py

@@ -452,6 +452,7 @@ def _init_core():
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
+            'ikvm': PluginRoute({'handler': 'ikvm'}),
         },
         'description': PluginRoute({
             'pluginattrs': ['hardwaremanagement.method'],

confluent_server/confluent/httpapi.py

@@ -69,6 +69,20 @@ opmap = {
 }
 
 
+def get_user_for_session(sessionid, sessiontok):
+    if not isinstance(sessionid, str):
+        sessionid = sessionid.decode()
+    if not isinstance(sessiontok, str):
+        sessiontok = sessiontok.decode()
+    if not sessiontok or not sessionid:
+        raise Exception("invalid session id or token")
+    if sessiontok != httpsessions.get(sessionid, {}).get('csrftoken', None):
+        raise Exception("Invalid csrf token for session")
+    user = httpsessions[sessionid]['name']
+    if not isinstance(user, str):
+        user = user.decode()
+    return user
+
 def group_creation_resources():
     yield confluent.messages.Attributes(
         kv={'name': None}, desc="Name of the group").html() + '<br>'

confluent_server/confluent/messages.py

@@ -262,10 +262,10 @@ class Generic(ConfluentMessage):
 
     def json(self):
         return json.dumps(self.data)
-    
+
     def raw(self):
         return self.data
-    
+
     def html(self):
         return json.dumps(self.data)
 
@@ -344,10 +344,10 @@ class ConfluentResourceCount(ConfluentMessage):
         self.myargs = [count]
         self.desc = 'Resource Count'
         self.kvpairs = {'count': count}
-    
+
     def strip_node(self, node):
         pass
-    
+
 class CreatedResource(ConfluentMessage):
     notnode = True
     readonly = True
@@ -569,6 +569,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputLicense(path, nodes, inputdata, configmanager)
     elif path == ['deployment', 'ident_image']:
         return InputIdentImage(path, nodes, inputdata)
+    elif path == ['console', 'ikvm']:
+        return InputIkvmParams(path, nodes, inputdata)
     elif inputdata:
         raise exc.InvalidArgumentException(
             'No known input handler for request')
@@ -941,6 +943,9 @@ class InputIdentImage(ConfluentInputMessage):
     keyname = 'ident_image'
     valid_values = ['create']
 
+class InputIkvmParams(ConfluentInputMessage):
+    keyname = 'method'
+    valid_values = ['unix', 'wss']
 
 class InputIdentifyMessage(ConfluentInputMessage):
     valid_values = set([

confluent_server/confluent/vinzmanager.py

@@ -0,0 +1,217 @@
+
+import confluent.auth as auth
+import eventlet
+import confluent.messages as msg
+import confluent.exceptions as exc
+import confluent.util as util
+import confluent.config.configmanager as configmanager
+import struct
+import eventlet.green.socket as socket
+import eventlet.green.subprocess as subprocess
+import base64
+import os
+import pwd
+import confluent.httpapi as httpapi
+mountsbyuser = {}
+_vinzfd = None
+_vinztoken = None
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+
+# Handle the vinz VNC session
+def assure_vinz():
+    global _vinzfd
+    global _vinztoken
+    if _vinzfd is None:
+        _vinztoken = base64.b64encode(os.urandom(33), altchars=b'_-').decode()
+        os.environ['VINZ_TOKEN'] = _vinztoken
+        os.makedirs('/var/run/confluent/vinz/sessions', exist_ok=True)
+
+        _vinzfd = subprocess.Popen(
+            ['/opt/confluent/bin/vinz',
+             '-c', '/var/run/confluent/vinz/control',
+             '-w', '127.0.0.1:4007',
+             '-a', '/var/run/confluent/vinz/approval',
+             # vinz supports unix domain websocket, however apache reverse proxy is dicey that way in some versions
+             '-d', '/var/run/confluent/vinz/sessions'])
+        while not os.path.exists('/var/run/confluent/vinz/control'):
+            eventlet.sleep(0.5)
+        eventlet.spawn(monitor_requests)
+
+_unix_by_nodename = {}
+def get_url(nodename, inputdata):
+    method = inputdata.inputbynode[nodename]
+    assure_vinz()
+    if method == 'wss':
+        return f'/vinz/kvmsession/{nodename}'
+    elif method == 'unix':
+        if nodename not in _unix_by_nodename or not os.path.exists(_unix_by_nodename[nodename]):
+            _unix_by_nodename[nodename] = request_session(nodename)
+        return _unix_by_nodename[nodename]
+
+
+_usersessions = {}
+def close_session(sessionid):
+    sessioninfo = _usersessions.get(sessionid, None)
+    if not sessioninfo:
+        return
+    del _usersessions[sessionid]
+    nodename = sessioninfo['nodename']
+    wc = sessioninfo['webclient']
+    cfg = configmanager.ConfigManager(None)
+    c = cfg.get_node_attributes(
+        nodename,
+        ['secret.hardwaremanagementuser',
+         'secret.hardwaremanagementpassword',
+        ], decrypt=True)
+    bmcuser = c.get(nodename, {}).get(
+        'secret.hardwaremanagementuser', {}).get('value', None)
+    bmcpass = c.get(nodename, {}).get(
+        'secret.hardwaremanagementpassword', {}).get('value', None)
+    if not isinstance(bmcuser, str):
+        bmcuser = bmcuser.decode()
+    if not isinstance(bmcpass, str):
+        bmcpass = bmcpass.decode()
+    if bmcuser and bmcpass:
+        wc.grab_json_response_with_status(
+            '/logout', {'data': [bmcuser, bmcpass]},
+            headers={
+                'Content-Type': 'application/json',
+                'Accept': 'application/json',
+                'X-XSRF-TOKEN': wc.cookies['XSRF-TOKEN']})
+
+
+def send_grant(conn, nodename):
+    cfg = configmanager.ConfigManager(None)
+    c = cfg.get_node_attributes(
+        nodename,
+        ['secret.hardwaremanagementuser',
+         'secret.hardwaremanagementpassword',
+         'hardwaremanagement.manager'], decrypt=True)
+    bmcuser = c.get(nodename, {}).get(
+        'secret.hardwaremanagementuser', {}).get('value', None)
+    bmcpass = c.get(nodename, {}).get(
+        'secret.hardwaremanagementpassword', {}).get('value', None)
+    bmc = c.get(nodename, {}).get(
+        'hardwaremanagement.manager', {}).get('value', None)
+    if bmcuser and bmcpass and bmc:
+        kv = util.TLSCertVerifier(cfg, nodename,
+                                  'pubkeys.tls_hardwaremanager').verify_cert
+        wc = webclient.SecureHTTPConnection(bmc, 443, verifycallback=kv)
+        if not isinstance(bmcuser, str):
+            bmcuser = bmcuser.decode()
+        if not isinstance(bmcpass, str):
+            bmcpass = bmcpass.decode()
+        rsp = wc.grab_json_response_with_status(
+            '/login', {'data': [bmcuser, bmcpass]},
+            headers={'Content-Type': 'application/json',
+                     'Accept': 'application/json'})
+        sessionid = wc.cookies['SESSION']
+        sessiontok = wc.cookies['XSRF-TOKEN']
+        _usersessions[sessionid] = {
+            'webclient': wc,
+            'nodename': nodename,
+        }
+        url = '/kvm/0'
+        fprintinfo = cfg.get_node_attributes(nodename, 'pubkeys.tls_hardwaremanager')
+        fprint = fprintinfo.get(
+            nodename, {}).get('pubkeys.tls_hardwaremanager', {}).get('value', None)
+        if not fprint:
+            return
+        fprint = fprint.split('$', 1)[1]
+        fprint = bytes.fromhex(fprint)
+        conn.send(struct.pack('!BI', 1, len(bmc)))
+        conn.send(bmc.encode())
+        conn.send(struct.pack('!I', len(sessionid)))
+        conn.send(sessionid.encode())
+        conn.send(struct.pack('!I', len(sessiontok)))
+        conn.send(sessiontok.encode())
+        conn.send(struct.pack('!I', len(fprint)))
+        conn.send(fprint)
+        conn.send(struct.pack('!I', len(url)))
+        conn.send(url.encode())
+        conn.send(b'\xff')
+
+def evaluate_request(conn):
+    allow = False
+    authname = None
+    try:
+        creds = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED,
+                                struct.calcsize('iII'))
+        pid, uid, gid = struct.unpack('iII', creds)
+        if uid != os.getuid():
+            return
+        rqcode, fieldlen = struct.unpack('!BI', conn.recv(5))
+        authtoken = conn.recv(fieldlen).decode()
+        if authtoken != _vinztoken:
+            return
+        if rqcode == 2:  # disconnect notification
+            fieldlen = struct.unpack('!I', conn.recv(4))[0]
+            sessionid = conn.recv(fieldlen).decode()
+            close_session(sessionid)
+            conn.recv(1)  # digest 0xff
+        if rqcode == 1:  # request for new connection
+            fieldlen = struct.unpack('!I', conn.recv(4))[0]
+            nodename = conn.recv(fieldlen).decode()
+            idtype = struct.unpack('!B', conn.recv(1))[0]
+            if idtype == 1:
+                usernum = struct.unpack('!I', conn.recv(4))[0]
+                if usernum == 0:  # root is a special guy
+                    send_grant(conn, nodename)
+                    return
+                try:
+                    authname = pwd.getpwuid(usernum).pw_name
+                except Exception:
+                    return
+            elif idtype == 2:
+                fieldlen = struct.unpack('!I', conn.recv(4))[0]
+                sessionid = conn.recv(fieldlen)
+                fieldlen = struct.unpack('!I', conn.recv(4))[0]
+                sessiontok = conn.recv(fieldlen)
+                try:
+                    authname = httpapi.get_user_for_session(sessionid, sessiontok)
+                except Exception:
+                    return
+            else:
+                return
+            conn.recv(1)  # should be 0xff
+            if authname:
+                allow = auth.authorize(authname, f'/nodes/{nodename}/console/ikvm')
+            if allow:
+                send_grant(conn, nodename)
+    finally:
+        conn.close()
+
+def monitor_requests():
+    a = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        os.remove('/var/run/confluent/vinz/approval')
+    except Exception:
+        pass
+    a.bind('/var/run/confluent/vinz/approval')
+    os.chmod('/var/run/confluent/vinz/approval', 0o600)
+    a.listen(8)
+    while True:
+        conn, addr = a.accept()
+        eventlet.spawn_n(evaluate_request, conn)
+
+def request_session(nodename):
+    assure_vinz()
+    a = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    a.connect('/var/run/confluent/vinz/control')
+    nodename = nodename.encode()
+    a.send(struct.pack('!BI', 1, len(nodename)))
+    a.send(nodename)
+    a.send(b'\xff')
+    rsp = a.recv(1)
+    retcode = struct.unpack('!B', rsp)[0]
+    if retcode != 1:
+        raise Exception("Bad return code")
+    rsp = a.recv(4)
+    nlen = struct.unpack('!I', rsp)[0]
+    sockname = a.recv(nlen).decode('utf8')
+    retcode = a.recv(1)
+    if retcode != b'\xff':
+        raise Exception("Unrecognized response")
+    return os.path.join('/var/run/confluent/vinz/sessions', sockname)
+

confluent_server/confluent_server.spec.tmpl

@@ -1,12 +1,16 @@
 %define name confluent_server
 %define version #VERSION#
+%define fversion %{lua:
+sv, _ = string.gsub("#VERSION#", "[~+]", "-")
+print(sv)
+}
 %define release 1
 
 Summary: confluent systems management server
 Name: %{name}
 Version: %{version}
 Release: %{release}
-Source0: %{name}-%{version}.tar.gz
+Source0: %{name}-%{fversion}.tar.gz
 License: Apache2
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
@@ -33,7 +37,7 @@ Url: https://github.com/lenovo/confluent
 Server for console management and systems management aggregation
 
 %prep
-%setup -n %{name}-%{version} -n %{name}-%{version}
+%setup -n %{name}-%{fversion}
 
 %build
 %if "%{dist}" == ".el7"

confluent_server/makesetup

@@ -2,7 +2,11 @@ cd `dirname $0`
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS+g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 echo $VERSION > VERSION
 sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py

confluent_vtbufferd/builddeb

@@ -8,7 +8,11 @@ DSCARGS="--with-python3=True --with-python2=False"
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 cd ..
 rm -rf /tmp/confluent

confluent_vtbufferd/buildrpm

@@ -1,7 +1,11 @@
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 mkdir -p dist/confluent_vtbufferd-$VERSION
 cp ../LICENSE NOTICE *.c *.h Makefile dist/confluent_vtbufferd-$VERSION

imgutil/builddeb

@@ -2,7 +2,11 @@
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 mkdir -p /tmp/confluent-imgutil
 cp -a * /tmp/confluent-imgutil

imgutil/buildrpm

@@ -2,7 +2,11 @@
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+    LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev)
+    LASTNUM=$((LASTNUM+1))
+    FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev)
+    VERSION=${FIRSTPART}.${LASTNUM}
+    VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3`
 fi
 sed -e "s/#VERSION#/$VERSION/" confluent_imgutil.spec.tmpl > confluent_imgutil.spec
 cp ../LICENSE .