Set nodename as hostname prior to post for el class distros

Initial collective join combined
with the orderly collective startup
hit a chicken and egg problem.

Disable initting on first enrollment
to let enrollment drive
that specific initialization to
restore behavior.

confluent_client/bin/nodeconfig

@@ -43,7 +43,7 @@ def bailout(msg, code=1):
     sys.exit(code)
 
 
-argparser = optparse.OptionParser(usage="Usage: %prog [options] <noderange> [setting|setting=value]")
+argparser = optparse.OptionParser(usage="Usage: %prog [options] noderange [option|option=value]")
 argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                      action='store_true', default=False,
                      help='Compare given settings to default or list settings '

confluent_client/bin/nodedeploy

@@ -89,23 +89,6 @@ def main(args):
         cleararm(args.noderange, c)
         clearpending(args.noderange, c)
     elif args.profile:
-        profnames = []
-        for prof in c.read('/deployment/profiles/'):
-            profname = prof.get('item', {}).get('href', None)
-            if profname:
-                profname = profname.replace('/', '')
-                profnames.append(profname)
-                if profname == args.profile:
-                    break
-        else:
-            sys.stderr.write('The specified profile "{}" is not an available profile\n'.format(args.profile))
-            if profnames:
-                sys.stderr.write('The following profiles are available:\n')
-                for profname in profnames:
-                    sys.stderr.write('    ' + profname + '\n')
-            else:
-                sys.stderr.write('No deployment profiles available, try osdeploy fiimport or imgutil capture\n')
-            sys.exit(1)
         armonce(args.noderange, c)
         setpending(args.noderange, args.profile, c)
     else:

confluent_client/bin/nodeeventlog

@@ -37,7 +37,7 @@ if sys.version_info[0] < 3:
     sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] <noderange> [clear]")
+    usage="Usage: %prog [options] noderange [clear]")
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to clear if clearing log, '

confluent_client/bin/nodefirmware

@@ -147,12 +147,10 @@ def update_firmware(session, filename):
 def show_firmware(session):
     global exitcode
     firmware_shown = False
-    nodes_matched = False
     for component in components:
         for res in session.read(
                 '/noderange/{0}/inventory/firmware/all/{1}'.format(
                     noderange, component)):
-            nodes_matched = True
             exitcode |= client.printerror(res)
             if 'databynode' not in res:
                 continue
@@ -163,9 +161,7 @@ def show_firmware(session):
                     for prefix in inv:
                         firmware_shown = True
                         printfirm(node, prefix, inv[prefix])
-    if not nodes_matched:
-        sys.stderr.write('No matching nodes for noderange "{0}"'.format(noderange))
-    elif not firmware_shown and not exitcode:
+    if not firmware_shown and not exitcode:
         argparser.print_help()
 
 

confluent_client/bin/nodelicense

@@ -36,7 +36,7 @@ exitcode = 0
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][install <filename>|save <directory>|delete <name>]")
+          "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to delete licenses from, '

confluent_client/bin/nodepower

@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] <noderange> "
+    usage="Usage: %prog [options] noderange "
           "([status|on|off|shutdown|boot|reset])")
 argparser.add_option('-p', '--showprevious', dest='previous',
                      action='store_true', default=False,
@@ -77,4 +77,4 @@ if options.previous:
     # add dictionary to session
     session.add_precede_dict(prev)
 
-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))
+sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))

confluent_client/bin/noderemove

@@ -33,7 +33,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='''\n       %prog <noderange>
+    usage='''\n       %prog noderange
              \n ''')
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '

confluent_client/bin/nodersync

@@ -40,7 +40,7 @@ import confluent.sortutil as sortutil
 
 def run():
     argparser = optparse.OptionParser(
-        usage="Usage: %prog <file/directorylist> <noderange>:<destination>",
+        usage="Usage: %prog location noderange:location",
     )
     argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '

confluent_client/bin/noderun

@@ -39,7 +39,7 @@ import confluent.sortutil as sortutil
 
 def run():
     argparser = optparse.OptionParser(
-        usage="Usage: %prog [options] <noderange> <command expression>",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,

confluent_client/confluent/client.py

@@ -507,7 +507,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                                                                   printattr)
                         else:
                             attrout = '{0}: {1}:'.format(node, printattr)
-                    elif isinstance(currattr, dict) and 'broken' in currattr:
+                    elif 'broken' in currattr:
                         attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
                                   '{2}'.format(node, printattr,
                                                currattr['broken'])
@@ -524,7 +524,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                         blame = options.blame
                     except AttributeError:
                         blame = False
-                    if blame or (isinstance(currattr, dict) and 'broken' in currattr):
+                    if blame or 'broken' in currattr:
                         blamedata = []
                         if 'inheritedfrom' in currattr:
                             blamedata.append('inherited from group {0}'.format(
@@ -633,7 +633,7 @@ def printgroupattributes(session, requestargs, showtype, nodetype, noderange, op
                         attrout = '{0}: {1}: ********'.format(noderange, attr)
                     else:
                         attrout = '{0}: {1}:'.format(noderange, attr)
-                elif isinstance(currattr, dict) and 'broken' in currattr:
+                elif 'broken' in currattr:
                     attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
                               '{2}'.format(noderange, attr,
                                            currattr['broken'])

confluent_client/confluent_env.sh

@@ -156,40 +156,6 @@ _confluent_osimage_completion()
     fi
 }
 
-_confluent_imgutil_completion()
-{
-    _confluent_get_args
-    if [ $NUMARGS == 2 ]; then
-        COMPREPLY=($(compgen -W "build exec unpack pack capture" -- ${COMP_WORDS[COMP_CWORD]}))
-        return
-    elif [ ${CMPARGS[1]} == 'build' ]; then
-        COMPREPLY=($(compgen -W "-s $(ls /var/lib/confluent/distributions)" -- ${COMP_WORDS[COMP_CWORD]}))
-	    return
-    elif [ ${CMPARGS[1]} == 'unpack' ]; then
-        if [ $NUMARGS == 3 ]; then
-            COMPREPLY=($(compgen -W "-s $(ls /var/lib/confluent/public/os)" -- ${COMP_WORDS[COMP_CWORD]}))
-            return
-        fi
-        compopt -o dirnames
-	    return
-    elif [ ${CMPARGS[1]} == 'pack' ]; then
-        if [ $NUMARGS == 3 ]; then
-            compopt -o dirnames
-        fi
-        COMPREPLY=()
-        return
-    elif [ ${CMPARGS[1]} == 'capture' ]; then
-        if [ $NUMARGS == 3 ]; then
-            _confluent_nn_completion
-            return
-        fi
-        return
-    elif [ ${CMPARGS[1]} == 'exec' ]; then
-        compopt -o dirnames
-	    COMPREPLY=($(compgen -W "-v" -- ${COMP_WORDS[COMP_CWORD]}))
-        return
-    fi
-}
 _confluent_nodedeploy_completion()
 {
     _confluent_get_args
@@ -272,6 +238,9 @@ _confluent_nodeattrib_completion()
 _confluent_nn_completion()
 {
     _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
     INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
@@ -333,7 +302,6 @@ complete -F _confluent_nr_completion nodeeventlog
 complete -F _confluent_nodefirmware_completion nodefirmware
 complete -F _confluent_nodedeploy_completion nodedeploy
 complete -F _confluent_osimage_completion osdeploy
-complete -F _confluent_imgutil_completion imgutil
 complete -F _confluent_ng_completion nodegroupattrib
 complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth

confluent_client/doc/man/collective.ronn

@@ -6,8 +6,8 @@ collective(1) -- Check and manage a confluent collective
 `collective invite <server>`  
 `collective join <server> [-i TOKEN]`  
 `collective show`  
-`collective gencert`  
-`collective delete`  
+`collective gencert`
+`collective delete`
 
 
 ## DESCRIPTION

confluent_client/doc/man/confluentdbutil.ronn

@@ -15,7 +15,7 @@ the json files (password protected, removed from the files, or unprotected).
 
 ## OPTIONS
 
-* `-p PASSWORD`, `--password=PASSWORD`:
+* `-p`, `--password`:
   If specified, information such as usernames and passwords will be encrypted
   using the given password.
   

confluent_client/doc/man/nodeconfig.ronn

@@ -27,7 +27,7 @@ actually be in effect until a reboot.
   If combined with `-x`, will show all differing values except those indicated
   by `-x`
   
-* `-b settings.batch`, `--batch=settings.batch`:
+* `-b`, `--batch`:
    Provide arguments as lines of a file, rather than the command line.  
    
 * `-d`, `--detail`:
@@ -47,7 +47,7 @@ actually be in effect until a reboot.
   Include advanced settings, which are normally not intended to be used
   without direction from the relevant server vendor.
 
-* `-r COMPONENT`, `--restoredefault=COMPONENT`:
+* `-r`, `--restoredefault`:
   Request that the specified component of the targeted nodes will have its
   configuration reset to default.  Currently the only component implemented
   is uefi.

confluent_client/doc/man/nodelicense.ronn

@@ -3,7 +3,7 @@ nodelicense(8) -- Manage license keys on BMC
 
 ## SYNOPSIS
 
-`nodelicense <noderange> [list][install <filename>|save <directory>|delete <license>]`
+`nodelicense <noderange> [list|install <filename>|delete <license>|save <directory>]`
 
 ## DESCRIPTION
 

confluent_client/doc/man/nodepower.ronn

@@ -3,7 +3,8 @@ nodepower(8) -- Check or change power state of confluent nodes
 
 ## SYNOPSIS
 
-`nodepower [options] <noderange> ([status|on|off|shutdown|boot|reset])`
+`nodepower <noderange>`  
+`nodepower <noderange> [-p] [on|off|boot|shutdown|reset|status]`
 
 ## DESCRIPTION
 
@@ -27,7 +28,7 @@ off will not react to this request.
 
 ## OPTIONS
 
-* `-p`, `--showprevious`:
+* `-p`, '--showprevious':
    Show previous power state for all directives that may change power state.
    
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:

confluent_client/doc/man/nodersync.ronn

@@ -12,7 +12,7 @@ noderange. This will present progress as percentage for all nodes.
 
 ## OPTIONS
 
-* `-f COUNT`, `-c COUNT`, `--count=COUNT`:
+* `-c`, `-f`, `--count`:
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
   active count completes. 

confluent_client/doc/man/noderun.ronn

@@ -3,7 +3,7 @@ noderun(8) -- Run arbitrary commands per node in a noderange
 
 ## SYNOPSIS
 
-`noderun [options] <noderange> <command expression>`
+`noderun <noderange> <command string>`
 
 ## DESCRIPTION
 
@@ -15,7 +15,7 @@ themselves, see nodeshell(8).
 
 ## OPTIONS
 
-* `-f COUNT`, `-c COUNT`, `--count=COUNT`:
+* `-f`, `-c`, `--count`:
   Number of commands to run at a time
   
 * `-n`, `--nonodeprefix`:

confluent_osdeploy/common/opt/confluent/bin/apiclient

@@ -8,32 +8,34 @@ import socket
 import subprocess
 import ssl
 import sys
-import time
-
-class InvalidApiKey(Exception):
-    pass
 
 def get_apikey(nodename, mgr):
-    apikey = ""
+    sealnew = True
     if os.path.exists('/etc/confluent/confluent.apikey'):
-        apikey = open('/etc/confluent/confluent.apikey').read().strip()
-        if apikey:
-            return apikey
-    while not apikey:
-        try:
-            apikey = subprocess.check_output(['/opt/confluent/bin/clortho', nodename, mgr])
-        except subprocess.CalledProcessError:
-            apikey = ''
-        if not isinstance(apikey, str):
-            apikey = apikey.decode('utf8')
-        if apikey.startswith('SEALED:'):
-            apikey = ""
-        if not apikey:
-            sys.stderr.write(
-                "Failed getting API token, check deployment.apiarmed attribute on {}\n".format(nodename))
-            time.sleep(10)
+        return open('/etc/confluent/confluent.apikey').read().strip()
+    apikey = subprocess.check_output(['/opt/confluent/bin/clortho', nodename, mgr])
+    if not isinstance(apikey, str):
+        apikey = apikey.decode('utf8')
+    if apikey.startswith('SEALED:'):
+        sealnew = False
+        with open('/etc/confluent/confluent.sealedapikey', 'w+') as apiout:
+            apiout.write(apikey[7:])
+        with open('/etc/confluent/confluent.sealedapikey') as inp:
+            sp = subprocess.Popen(['/usr/bin/clevis-decrypt-tpm2'],
+                stdin=inp, stdout=subprocess.PIPE)
+            apikey = sp.communicate()[0]
+            if not isinstance(apikey, str):
+                apikey = apikey.decode('utf8')
     with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
         apiout.write(apikey)
+    if sealnew and os.path.exists('/usr/bin/clevis-encrypt-tpm2'):
+        try:
+            with open('/etc/confluent/confluent.apikey') as apin:
+                sealed = subprocess.check_output(
+                    ['/usr/bin/clevis-encrypt-tpm2', '{}'], stdin=apin)
+            print(HTTPSClient().grab_url('/confluent-api/self/saveapikey', sealed).decode())
+        except Exception:
+            sys.stderr.write('Unable to persist API key through TPM2 sealing\n')
     apikey = apikey.strip()
     os.chmod('/etc/confluent/confluent.apikey', 0o600)
     return apikey
@@ -77,8 +79,6 @@ class HTTPSClient(client.HTTPConnection, object):
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
         client.HTTPConnection.__init__(self, host, port)
-        self.host = host
-        self.node = node
         self.connect()
 
     def set_header(self, key, val):
@@ -107,23 +107,13 @@ class HTTPSClient(client.HTTPConnection, object):
             method = 'POST'
         else:
             method = 'GET'
-        authed = False
-        while not authed:
-            authed = True
-            self.request(method, url, data, headers=self.stdheaders)
-            rsp = self.getresponse()
-            if rsp.status >= 200 and rsp.status < 300:
-                if returnrsp:
-                    return rsp.status, rsp
-                else:
-                    return rsp.status, rsp.read()
-            if rsp.status == 401:
-                authed = False
-                rsp.read()
-                with open('/etc/confluent/confluent.apikey', 'w+') as akfile:
-                    akfile.write('')
-                self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(
-                    self.node, self.host)
+        self.request(method, url, data, headers=self.stdheaders)
+        rsp = self.getresponse()
+        if rsp.status >= 200 and rsp.status < 300:
+            if returnrsp:
+                return rsp.status, rsp
+            else:
+                return rsp.status, rsp.read()
         raise Exception(rsp.read())
 
 if __name__ == '__main__':

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -4,7 +4,7 @@ Release: 1
 Summary: OS Deployment support for confluent
 
 License: Apache2
-URL: https://hpc.lenovo.com/
+URL: http://hpc.lenovo.com/
 Source0: confluent_osdeploy.tar.xz
 BuildArch: noarch
 Requires: confluent_ipxe mtools tar
@@ -21,13 +21,11 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 
 %build
 mkdir -p opt/confluent/bin
-mkdir -p stateless-bin
 cd utils
 make all
-cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
-cp start_root urlmount ../stateless-bin/
+cp copernicus clortho autocons ../opt/confluent/bin
 cd ..
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos; do
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04; do
     mkdir ${os}out
     cd ${os}out
     cp -a ../opt .
@@ -37,17 +35,6 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos; do
     mv ../addons.cpio .
     cd ..
 done
-for os in el8 suse15; do
-    mkdir ${os}disklessout
-    cd ${os}disklessout
-    cp -a ../opt .
-    cp -a ../${os}-diskless/initramfs/* .
-    cp -a ../common/* .
-    cp -a ../stateless-bin/* opt/confluent/bin
-    find . | cpio -H newc -o > ../addons.cpio
-    mv ../addons.cpio .
-    cd ..
-done
 mkdir esxi7out
 cd esxi7out
 cp -a ../opt .
@@ -61,19 +48,13 @@ cp -a esxi7out esxi6out
 cp -a esxi7 esxi6
 
 %install
-for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7 coreos; do
+for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     cp -a $os/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
-    if [ -d ${os}disklessout ]; then
-        mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs
-        mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/profiles
-        cp ${os}disklessout/addons.* %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs
-        cp -a ${os}-diskless/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os-diskless/profiles
-    fi
+    find %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles -name .gitignore -exec rm -f {} +
 done
-find %{buildroot}/opt/confluent/lib/osdeploy/ -name .gitignore -exec rm -f {} +
 
 %files
 /opt/confluent/lib/osdeploy

confluent_osdeploy/coreos/initramfs/etc/systemd/system/initrd-root-fs.target.requires/confluent-rootfs.service

@@ -1 +0,0 @@
-/usr/lib/systemd/system/confluent-rootfs.service

confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh

@@ -1,106 +0,0 @@
-#!/bin/bash
-TRIES=0
-oum=$(umask)
-umask 0077
-mkdir -p /etc/confluent
-echo -n > /etc/confluent/confluent.info
-umask $oum
-cd /sys/class/net
-while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
-    TRIES=$((TRIES + 1))
-    for currif in *; do
-        ip link set $currif up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
-done
-cd /
-grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
-echo -n "" > /tmp/confluent.initq
-# restart cmdline
-echo -n "" > /etc/cmdline.d/01-confluent.conf
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-cat /tls/*.pem > /etc/confluent/ca.pem
-confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
-if [[ $confluent_mgr == *%* ]]; then
-    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
-    ifidx=$(cat /tmp/confluent.ifidx)
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-fi
-needseal=1
-oldumask=$(umask)
-umask 0077
-while [ -z "$confluent_apikey" ]; do
-    /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
-    if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
-	needseal=0
-        sed -e s/^SEALED:// /etc/confluent/confluent.apikey | clevis-decrypt-tpm2 > /etc/confluent/confluent.apikey.decrypt
-        mv /etc/confluent/confluent.apikey.decrypt /etc/confluent/confluent.apikey
-    fi
-    confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-    if [ -z "$confluent_apikey" ]; then
-        echo "Unable to acquire node api key, no TPM2 sealed nor fresh token available, retrying..."
-        sleep 10
-    fi
-done
-if [[ $confluent_mgr == *:* ]]; then
-    confluent_mgr="[$confluent_mgr]"
-fi
-if [ $needseal == 1 ]; then
-    sealed=$(echo $confluent_apikey | clevis-encrypt-tpm2 {})
-    if [ ! -z "$sealed" ]; then
-        curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d $sealed https://$confluent_mgr/confluent-api/self/saveapikey
-    fi
-fi
-curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-umask $oldumask
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
-autoconfigmethod=${autoconfigmethod#ipv4_method: }
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
-confluent_profile=${confluent_profile#profile: }
-
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
-    fi
-    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#ipv4_netmask: }
-    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
-fi
-nameserversec=0
-while read -r entry; do
-    if [ $nameserversec = 1 ]; then
-        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
-            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
-            continue
-        fi
-    fi
-    nameserversec=0
-    if [ "${entry%:*}" = "nameservers" ]; then
-        nameserversec=1
-        continue
-    fi
-done < /etc/confluent/confluent.deploycfg
-
-if [ -e /lib/nm-lib.sh ]; then
-    . /lib/nm-lib.sh
-    nm_generate_connections
-    if [[ "$ifname" == ib* ]]; then
-        sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
-        if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then
-            echo >> /run/NetworkManager/system-connections/$ifname.nmconnection
-            echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
-            echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection
-        fi
-    fi
-fi
-cat /proc/cmdline /etc/cmdline.d/01-confluent.conf | tr '\n' ' ' > /run/fakecmdline
-mount -o bind /run/fakecmdline /proc/cmdline
-
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/rootfs.img | rdcore stream-hash /etc/coreos-live-want-rootfs | bsdtar -xf - -C /

confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -1,10 +0,0 @@
-cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
-if ! grep console= /proc/cmdline >& /dev/null; then
-    autocons=$(/opt/confluent/bin/autocons)
-    if [ -n "$autocons" ]; then
-        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
-        autocons=${autocons%,*}
-	echo $autocons > /tmp/01-autocons.devnode
-	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
-    fi
-fi

confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-# if ignition configuration is driving, defer to it
-if [ ! -f /config.ign ]; then
-
-    nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info| head -n 1| awk '{print $2}' | sed -e s/%/%25/)
-    if [[ $confluent_mgr = *:* ]]; then
-        confluent_mgr=[$confluent_mgr]
-    fi
-    rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
-    rootpassword=${rootpassword#rootpassword: }
-    if [ "$rootpassword" = "null" ]; then
-        rootpassword=""
-    fi
-
-    if [ ! -z "$rootpassword" ]; then
-        sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
-    fi
-
-    mount -o bind /dev /sysroot/dev
-    chroot /sysroot ssh-keygen -A
-    umount /sysroot/dev
-    for i in  /sysroot/etc/ssh/ssh_host*key.pub; do
-        certname=${i/.pub/-cert.pub}
-        curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname
-        echo HostKey ${i%.pub} | sed -e 's!/sysroot!!' >> /sysroot/etc/ssh/sshd_config
-        echo HostCertificate $certname | sed -e 's!/sysroot!!' >> /sysroot/etc/ssh/sshd_config
-done

confluent_osdeploy/coreos/initramfs/usr/lib/systemd/system/confluent-rootfs.service

@@ -1,19 +0,0 @@
-[Unit]
-Description=Confluent initialization
-DefaultDependencies=false
-
-After=basic.target
-# Network is enabled here
-After=dracut-initqueue.service
-Before=coreos-livepxe-rootfs.service
-
-
-# If we fail, the boot will fail.  Be explicit about it.
-OnFailure=emergency.target
-OnFailureJobMode=isolate
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/opt/confluent/bin/initconfluent.sh
-

confluent_osdeploy/coreos/profiles/default/initprofile.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
-ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution && \
-ln -s $1/images/ignition.img $2/boot/initramfs/ignition.img && \
-mkdir -p $2/boot/efi/boot/ && \
-ln -s $1/images/pxeboot/rootfs.img $2/ && \
-mcopy -i $1/images/efiboot.img ::efi/redhat/grubx64.efi $2/boot/efi/boot/ && \
-mcopy -i $1/images/efiboot.img ::efi/boot/bootx64.efi $2/boot/efi/boot/

confluent_osdeploy/coreos/profiles/default/profile.yaml

@@ -1,5 +0,0 @@
-label: RedHat CoreOS %%VERSION%% %%ARCH%% (Default Profile)
-kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal
-#the above boots and will have ignition agent phone home and await instructions
-#kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal  coreos.inst=yes coreos.inst.install_dev=sda 
-#the above runs the installer to write to disk

confluent_osdeploy/el7/profiles/default/kickstart

@@ -35,11 +35,10 @@ reboot
 %packages
 #-kernel-uek # This can opt out of the UEK for the relevant distribution
 chrony
-pciutils
-python
 rsync
+python
 tar
--iwl*-firmware
+pciutils
 %include /tmp/addonpackages
 %end
 

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -11,9 +11,6 @@ confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{prin
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
-	sleep 1
-done
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -1,7 +1,4 @@
 function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
     if [ -z "$confluent_mgr" ]; then
         confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
@@ -54,7 +51,7 @@ source_remote() {
     chmod +x $1
     cmd=$1
     shift
-    source ./$cmd
+    source $cmd
     cd - > /dev/null
     return $retcode
 }

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -1,240 +0,0 @@
-get_remote_apikey() {
-    while [ -z "$confluent_apikey" ]; do
-        /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
-        if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
-            # we don't support remote sealed api keys anymore
-            echo > /etc/confluent/confluent.apikey
-        fi
-        confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-        if [ -z "$confluent_apikey" ]; then
-            echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
-            sleep 10
-        else
-            tmpdir=$(mktemp -d)
-            cd $tmpdir
-            tpm2_startauthsession --session=session.ctx
-            tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
-            tpm2_createprimary -G ecc -Q --key-context=prim.ctx
-            (echo -n "CONFLUENT_APIKEY:";cat /etc/confluent/confluent.apikey) | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
-            tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
-            tpm2_evictcontrol -Q -c data.ctx
-            tpm2_flushcontext session.ctx
-            cd - > /dev/null
-            rm -rf $tmpdir
-        fi
-    done
-}
-root=1
-rootok=1
-netroot=confluent
-clear
-mkdir -p /etc/ssh
-mkdir -p /var/tmp/
-mkdir -p /var/empty/sshd
-mkdir -p /etc/confluent
-sed -i '/^root:x/d' /etc/passwd
-echo root:x:0:0::/:/bin/bash >> /etc/passwd
-echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
-
-if ! grep console= /proc/cmdline >& /dev/null; then
-    autocons=$(/opt/confluent/bin/autocons)
-    autocons=${autocons##*/}
-    echo "Automatic console configured for $autocons"
-fi
-echo "Initializing confluent diskless environment"
-echo -n "udevd: "
-/usr/lib/systemd/systemd-udevd --daemon
-echo -n "Loading drivers..."
-udevadm trigger
-udevadm trigger --type=devices --action=add
-udevadm settle
-modprobe ib_ipoib
-modprobe ib_umad
-modprobe hfi1
-modprobe mlx5_ib
-echo "done"
-cat > /etc/ssh/sshd_config << EOF
-Port 2222
-Subsystem       sftp    /usr/libexec/openssh/sftp-server
-PermitRootLogin yes
-AuthorizedKeysFile      .ssh/authorized_keys
-EOF
-mkdir /root/.ssh
-mkdir /.ssh
-cat /ssh/*pubkey > /root/.ssh/authorized_keys 2>/dev/null
-cp /root/.ssh/authorized_keys /.ssh/
-cat /tls/*.pem > /etc/confluent/ca.pem
-mkdir -p /etc/pki/tls/certs
-cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
-TRIES=0
-oldumask=$(umask)
-umask 0077
-tpmdir=$(mktemp -d)
-cd $tpmdir
-lasthdl=""
-for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
-    tpm2_startauthsession --policy-session --session=session.ctx
-    tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
-    unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
-    tpm2_flushcontext session.ctx
-    if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
-        confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
-        echo $confluent_apikey > /etc/confluent/confluent.apikey
-        if [ -n "$lasthdl" ]; then
-            tpm2_evictcontrol -c $lasthdl
-        fi
-        lasthdl=$hdl
-    fi
-done
-cd - > /dev/null
-rm -rf $tpmdir
-touch /etc/confluent/confluent.info
-cd /sys/class/net
-echo -n "Scanning for network configuration..."
-while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
-    TRIES=$((TRIES + 1))
-    for i in *; do
-        ip link set $i up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
-done
-cd /
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-hostname $nodename
-confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1 | awk -F': ' '{print $2}' | awk -F'|' '{print $1}')
-if [ -z "$confluent_mgr" ]; then
-    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
-fi
-if [[ $confluent_mgr == *%* ]]; then
-    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
-    ifidx=$(cat /tmp/confluent.ifidx)
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-fi
-
-ready=0
-while [ $ready = "0" ]; do
-    get_remote_apikey
-    if [[ $confluent_mgr == *:* ]]; then
-        confluent_mgr="[$confluent_mgr]"
-    fi
-    tmperr=$(mktemp)
-    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
-    if grep 401 $tmperr > /dev/null; then
-        confluent_apikey=""
-        if [ -n "$lasthdl" ]; then
-            tpm2_evictcontrol -c $lasthdl
-        fi
-        confluent_mgr=${confluent_mgr#[}
-        confluent_mgr=${confluent_mgr%]}
-    else
-        ready=1
-    fi
-    rm $tmperr
-done
-tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
-umask $oldumask
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg |awk '{print $2}')
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo -n "Attempting to use dhcp to bring up $ifname..."
-    dhclient $ifname
-    echo "Complete:"
-    ip addr show dev $ifname
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
-    fi
-    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#prefix: }
-    echo "Setting up $ifname as static at $v4addr/$v4nm"
-    ip addr add dev $ifname $v4addr/$v4nm
-    if [ ! -z "$v4gw" ]; then
-        ip route add default via $v4gw
-    fi
-    mkdir -p /run/NetworkManager/system-connections
-    cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-[connection]
-EOC
-    echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
-autoconnect-retries=1
-EOC
-    echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-multi-connect=1
-permissions=
-wait-device-timeout=60000
-
-[ethernet]
-mac-address-blacklist=
-
-[ipv4]
-EOC
-    echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    if [ ! -z "$v4gw" ]; then
-        echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    fi
-    nameserversec=0
-    nameservers=""
-    while read -r entry; do
-        if [ $nameserversec = 1 ]; then
-            if [[ $entry == "-"* ]]; then
-                nameservers="$nameservers"${entry#- }";"
-                continue
-            fi
-        fi
-        nameserversec=0
-        if [ "${entry%:*}" = "nameservers" ]; then
-            nameserversec=1
-            continue
-        fi
-    done < /etc/confluent/confluent.deploycfg
-    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
-    dnsdomain=${dnsdomain#dnsdomain: }
-    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-may-fail=false
-method=manual
-
-[ipv6]
-addr-gen-mode=eui64
-method=auto
-
-[proxy]
-EOC
-fi
-chmod 600 /run/NetworkManager/system-connections/*.nmconnection
-echo -n "Initializing ssh..."
-ssh-keygen -A
-for pubkey in /etc/ssh/ssh_host*key.pub; do
-    certfile=${pubkey/.pub/-cert.pub}
-    privfile=${pubkey%.pub}
-    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
-    if [ -s $certfile ]; then
-        echo HostCertificate $certfile >> /etc/ssh/sshd_config
-    fi
-    echo HostKey $privfile >> /etc/ssh/sshd_config
-done
-/usr/sbin/sshd
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
-confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
-confluent_urls=""
-for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed -e s/%/%25/); do
-    if [[ $addr == *:* ]]; then
-        confluent_urls="$confluent_urls $confluent_proto://[$addr]/confluent-public/os/$confluent_profile/rootimg.sfs"
-    else
-        confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
-    fi
-done
-confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
-mkdir -p /etc/confluent
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
-. /etc/confluent/functions
-source_remote imageboot.sh

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.custom

@@ -1,4 +0,0 @@
-. /etc/confluent/functions
-# This is a convenient place to keep customizations separate from modifying the stock scripts
-# While modification of the stock scripts is fine, it may be easier to rebase to a newer
-# stock profile if the '.custom' files are used.

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=First Boot Process
-Requires=network-online.target
-After=network-online.target
-
-[Service]
-ExecStart=/opt/confluent/bin/firstboot.sh
-
-[Install]
-WantedBy=multi-user.target
-

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -1,39 +0,0 @@
-#!/bin/sh
-
-# This script is executed on the first boot after install has
-# completed. It is best to edit the middle of the file as
-# noted below so custom commands are executed before
-# the script notifies confluent that install is fully complete.
-
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
-export nodename confluent_mgr confluent_profile
-. /etc/confluent/functions
-exec >> /var/log/confluent/confluent-firstboot.log
-exec 2>> /var/log/confluent/confluent-firstboot.log
-tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
-logshowpid=$!
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
-	sleep 1
-done
-
-if [ ! -f /etc/confluent/firstboot.ran ]; then
-    touch /etc/confluent/firstboot.ran
-
-    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
-
-    run_remote firstboot.custom
-    # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
-    run_remote_parts firstboot.d
-
-    # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
-    run_remote_config firstboot.d
-fi
-
-curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
-systemctl disable firstboot
-rm /etc/systemd/system/firstboot.service
-rm /etc/confluent/firstboot.ran
-kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/functions

@@ -1,144 +0,0 @@
-function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
-    if [ -z "$confluent_mgr" ]; then
-        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
-    fi
-    if [ -z "$confluent_profile" ]; then
-        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
-    fi
-}
-
-fetch_remote() {
-    curlargs=""
-    if [ -f /etc/confluent/ca.pem ]; then
-        curlargs=" --cacert /etc/confluent/ca.pem"
-    fi
-    set_confluent_vars
-    mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
-    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
-}
-
-source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d)
-    apiclient=/opt/confluent/bin/apiclient
-    if [ -f /etc/confluent/apiclient ]; then
-        apiclient=/etc/confluent/apiclient
-    fi
-    scriptlist=$(/usr/libexec/platform-python $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
-    for script in $scriptlist; do
-        source_remote $1/$script
-    done
-    unset confluentscripttmpdir
-}
-
-run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d)
-    apiclient=/opt/confluent/bin/apiclient
-    if [ -f /etc/confluent/apiclient ]; then
-        apiclient=/etc/confluent/apiclient
-    fi
-    scriptlist=$(/usr/libexec/platform-python $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
-    for script in $scriptlist; do
-        run_remote $1/$script
-    done
-    unset confluentscripttmpdir
-}
-
-source_remote() {
-    set_confluent_vars
-    echo
-    echo '---------------------------------------------------------------------------'
-    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d)
-    fi
-    echo Sourcing from $confluentscripttmpdir
-    cd $confluentscripttmpdir
-    fetch_remote $1
-    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
-    chmod +x $1
-    cmd=$1
-    shift
-    source ./$cmd
-    cd - > /dev/null
-    return $retcode
-}
-
-run_remote() {
-    requestedcmd="'$*'"
-    set_confluent_vars
-    echo
-    echo '---------------------------------------------------------------------------'
-    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d)
-    fi
-    echo Executing in $confluentscripttmpdir
-    cd $confluentscripttmpdir
-    fetch_remote $1
-    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
-    chmod +x $1
-    cmd=$1
-    if [ -x /usr/bin/chcon ]; then
-        chcon system_u:object_r:bin_t:s0 $cmd
-    fi
-    shift
-    ./$cmd $*
-    retcode=$?
-    if [ $retcode -ne 0 ]; then
-        echo "$requestedcmd exited with code $retcode"
-    fi
-    cd - > /dev/null
-    return $retcode
-}
-
-run_remote_python() {
-    echo
-    set_confluent_vars
-    if [ -f /etc/confluent/ca.pem ]; then
-        curlargs=" --cacert /etc/confluent/ca.pem"
-    fi
-    echo '---------------------------------------------------------------------------'
-    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    tmpdir=$(mktemp -d)
-    echo Executing in $tmpdir
-    cd $tmpdir
-    mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
-    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
-    if [ -x /usr/libexec/platform-python ]; then
-        /usr/libexec/platform-python $*
-    else
-        /usr/bin/python $*
-    fi
-    retcode=$?
-    echo "'$*' exited with code $retcode"
-    cd - > /dev/null
-    return $retcode
-}
-
-run_remote_config() {
-    echo
-    set_confluent_vars
-    apiclient=/opt/confluent/bin/apiclient
-    if [ -f /etc/confluent/apiclient ]; then
-        apiclient=/etc/confluent/apiclient
-    fi
-    echo '---------------------------------------------------------------------------'
-    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
-    if [ -x /usr/libexec/platform-python ]; then
-        /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
-        /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/status -w 204
-    else
-        /usr/bin/python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
-        /usr/bin/python $apiclient /confluent-api/self/remoteconfig/status -w 204
-    fi
-    echo
-    echo 'Completed remote configuration'
-    echo '---------------------------------------------------------------------------'
-    return
-}
-

confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk

@@ -1,93 +0,0 @@
-import subprocess
-import os
-
-class DiskInfo(object):
-    def __init__(self, devname):
-        self.name = devname
-        self.wwn = None
-        self.path = None
-        self.model = ''
-        self.size = 0
-        self.driver = None
-        self.mdcontainer = ''
-        devnode = '/dev/{0}'.format(devname)
-        qprop = subprocess.check_output(
-            ['udevadm', 'info', '--query=property', devnode])
-        if not isinstance(qprop, str):
-            qprop = qprop.decode('utf8')
-        for prop in qprop.split('\n'):
-            if '=' not in prop:
-                continue
-            k, v = prop.split('=', 1)
-            if k == 'DEVTYPE' and v != 'disk':
-                raise Exception('Not a disk')
-            elif k == 'DM_NAME':
-                raise Exception('Device Mapper')
-            elif k == 'ID_MODEL':
-                self.model = v
-            elif k == 'DEVPATH':
-                self.path = v
-            elif k == 'ID_WWN':
-                self.wwn = v
-            elif k == 'MD_CONTAINER':
-                self.mdcontainer = v
-        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
-        if not isinstance(attrs, str):
-            attrs = attrs.decode('utf8')
-        for attr in attrs.split('\n'):
-            if '==' not in attr:
-                continue
-            k, v = attr.split('==', 1)
-            k = k.strip()
-            if k == 'ATTRS{size}':
-                self.size = v.replace('"', '')
-            elif (k == 'DRIVERS' and not self.driver
-                    and v not in ('"sd"', '""')):
-                self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
-            raise Exception("No driver detected")
-        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
-            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
-                self.size = int(sizesrc.read()) * 512
-        if int(self.size) < 536870912:
-            raise Exception("Device too small for install")
-
-    @property
-    def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
-            return 0
-        if 'imsm' in self.mdcontainer:
-            return 1
-        if self.driver == 'ahci':
-            return 2
-        if self.driver.startswith('megaraid'):
-            return 3
-        if self.driver.startswith('mpt'):
-            return 4
-        return 99
-
-    def __repr__(self):
-        return repr({
-            'name': self.name,
-            'path': self.path,
-            'wwn': self.wwn,
-            'driver': self.driver,
-            'size': self.size,
-            'model': self.model,
-        })
-
-
-def main():
-    disks = []
-    for disk in sorted(os.listdir('/sys/class/block')):
-        try:
-            disk = DiskInfo(disk)
-            disks.append(disk)
-        except Exception as e:
-            print("Skipping {0}: {1}".format(disk, str(e)))
-    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
-    if nd:
-        open('/tmp/installdisk', 'w').write(nd[0])
-
-if __name__ == '__main__':
-    main()

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -1,376 +0,0 @@
-#!/usr/bin/python3
-import glob
-import json
-import os
-import re
-import time
-import shutil
-import socket
-import stat
-import struct
-import sys
-import subprocess
-
-def get_next_part_meta(img, imgsize):
-    if img.tell() == imgsize:
-        return None
-    pathlen = struct.unpack('!H', img.read(2))[0]
-    mountpoint = img.read(pathlen).decode('utf8')
-    jsonlen = struct.unpack('!I', img.read(4))[0]
-    metadata = json.loads(img.read(jsonlen).decode('utf8'))
-    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
-    nextlen = struct.unpack('!H', img.read(2))[0]
-    img.seek(nextlen, 1) # skip filesystem type
-    nextlen = struct.unpack('!H', img.read(2))[0]
-    img.seek(nextlen, 1) # skip orig devname (redundant with json)
-    nextlen = struct.unpack('!H', img.read(2))[0]
-    img.seek(nextlen, 1) # skip padding
-    nextlen = struct.unpack('!Q', img.read(8))[0]
-    img.seek(nextlen, 1)  # go to next section
-    return metadata
-
-def get_multipart_image_meta(img):
-    img.seek(0, 2)
-    imgsize = img.tell()
-    img.seek(16)
-    seekamt = img.read(1)
-    img.seek(struct.unpack('B', seekamt)[0], 1)
-    partinfo = get_next_part_meta(img, imgsize)
-    while partinfo:
-        yield partinfo
-        partinfo = get_next_part_meta(img, imgsize)
-
-def get_image_metadata(imgpath):
-    with open(imgpath, 'rb') as img:
-        header = img.read(16)
-        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
-            for md in get_multipart_image_meta(img):
-                yield md
-        else:
-            raise Exception('Installation from single part image not supported')
-
-class PartedRunner():
-    def __init__(self, disk):
-        self.disk = disk
-
-    def run(self, command):
-        command = command.split()
-        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
-
-def fixup(rootdir, vols):
-    devbymount = {}
-    for vol in vols:
-        devbymount[vol['mount']] = vol['targetdisk']
-    fstabfile = os.path.join(rootdir, 'etc/fstab')
-    with open(fstabfile) as tfile:
-        fstab = tfile.read().split('\n')
-    while not fstab[0]:
-        fstab = fstab[1:]
-    if os.path.exists(os.path.join(rootdir, '.autorelabel')):
-        os.unlink(os.path.join(rootdir, '.autorelabel'))
-    with open(fstabfile, 'w') as tfile:
-        for tab in fstab:
-            entry = tab.split()
-            if tab.startswith('#ORIGFSTAB#'):
-                if entry[1] in devbymount:
-                    targetdev = devbymount[entry[1]]
-                    if targetdev.startswith('/dev/localstorage/'):
-                        entry[0] = targetdev
-                    else:
-                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
-                        uuid = uuid.strip()
-                        entry[0] = 'UUID={}'.format(uuid)
-                elif entry[2] == 'swap':
-                    entry[0] = '/dev/mapper/localstorage-swap'
-                entry[0] = entry[0].ljust(42)
-                entry[1] = entry[1].ljust(16)
-                entry[3] = entry[3].ljust(28)
-                tab = '\t'.join(entry)
-            tfile.write(tab + '\n')
-    with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout:
-        nameout.write(socket.gethostname() + '\n')
-    selinuxconfig = os.path.join(rootdir, 'etc/selinux/config')
-    policy = None
-    if os.path.exists(selinuxconfig):
-        with open(selinuxconfig) as cfgin:
-            sec = cfgin.read().split('\n')
-            for l in sec:
-                l = l.split('#', 1)[0]
-                if l.startswith('SELINUXTYPE='):
-                    _, policy = l.split('=')
-    for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')):
-        os.unlink(sshkey)
-    for sshkey in glob.glob('/etc/ssh/*_key*'):
-        newkey = os.path.join(rootdir, sshkey[1:])
-        shutil.copy2(sshkey, newkey)
-        finfo = os.stat(sshkey)
-        os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID])
-    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/sysconfig/network-scripts/*')):
-        os.unlink(ifcfg)
-    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/NetworkManager/system-connections/*')):
-        os.unlink(ifcfg)
-    for ifcfg in glob.glob('/run/NetworkManager/system-connections/*'):
-        newcfg = ifcfg.split('/')[-1]
-        newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg))
-        shutil.copy2(ifcfg, newcfg)
-    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
-    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
-    if policy:
-        sys.stdout.write('Applying SELinux labeling...')
-        sys.stdout.flush()
-        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')])
-        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')])
-        sys.stdout.write('Done\n')
-        sys.stdout.flush()
-    for metafs in ('proc', 'sys', 'dev'):
-        subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
-        defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
-        for gline in defgrub:
-            gline = gline.split()
-            newline = []
-            for ent in gline:
-                if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
-                    continue
-                newline.append(ent)
-            defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
-    newroot = None
-    with open('/etc/shadow') as shadowin:
-        shents = shadowin.read().split('\n')
-        for shent in shents:
-            shent = shent.split(':')
-            if not shent:
-                continue
-            if shent[0] == 'root' and shent[1] not in ('*', '!!', ''):
-                newroot = shent[1]
-    if newroot:
-        shlines = None
-        with open(os.path.join(rootdir, 'etc/shadow')) as oshadow:
-            shlines = oshadow.read().split('\n')
-        with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow:
-            for line in shlines:
-                if line.startswith('root:'):
-                    line = line.split(':')
-                    line[1] = newroot
-                    line = ':'.join(line)
-                oshadow.write(line + '\n')
-    partnum = None
-    targblock = None
-    for vol in vols:
-        if vol['mount'] == '/boot/efi':
-            targdev = vol['targetdisk']
-            partnum = re.search('(\d+)$', targdev).group(1)
-            targblock = re.search('(.*)\d+$', targdev).group(1)
-    if targblock:
-        shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
-        shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
-        subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
-    #other network interfaces
-
-
-def had_swap():
-    with open('/etc/fstab') as tabfile:
-        tabs = tabfile.read().split('\n')
-        for tab in tabs:
-            tab = tab.split()
-            if len(tab) < 3:
-                continue
-            if tab[2] == 'swap':
-                return True
-    return False
-
-def install_to_disk(imgpath):
-    lvmvols = {}
-    deftotsize = 0
-    mintotsize = 0
-    deflvmsize = 0
-    minlvmsize = 0
-    biggestsize = 0
-    biggestfs = None
-    plainvols = {}
-    allvols = []
-    swapsize = 0
-    if had_swap():
-        with open('/proc/meminfo') as meminfo:
-            swapsize = meminfo.read().split('\n')[0]
-        swapsize = int(swapsize.split()[1])
-        if swapsize < 2097152:
-            swapsize = swapsize * 2
-        elif swapsize > 8388608 and swapsize < 67108864:
-            swapsize = swapsize * 0.5
-        elif swapsize >= 67108864:
-            swapsize = 33554432
-        swapsize = int(swapsize * 1024)
-    deftotsize = swapsize
-    mintotsize = swapsize
-    for fs in get_image_metadata(imgpath):
-        allvols.append(fs)
-        deftotsize += fs['initsize']
-        mintotsize += fs['minsize']
-        if fs['initsize'] > biggestsize:
-            biggestfs = fs
-            biggestsize = fs['initsize']
-        if fs['device'].startswith('/dev/mapper'):
-            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
-            deflvmsize += fs['initsize']
-            minlvmsize += fs['minsize']
-        else:
-            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
-    with open('/tmp/installdisk') as diskin:
-        instdisk = diskin.read()
-    instdisk = '/dev/' + instdisk
-    parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
-    dinfo = dinfo.split('\n')
-    sectors = 0
-    sectorsize = 0
-    for inf in dinfo:
-        if inf.startswith('Disk {0}:'.format(instdisk)):
-            _, sectors = inf.split(': ')
-            sectors = int(sectors.replace('s', ''))
-        if inf.startswith('Sector size (logical/physical):'):
-            _, sectorsize = inf.split(':')
-            sectorsize = sectorsize.split('/')[0]
-            sectorsize = sectorsize.replace('B', '')
-            sectorsize = int(sectorsize)
-    # for now, only support resizing/growing the largest partition
-    minexcsize = deftotsize - biggestfs['initsize']
-    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
-    minsectors = mintotsize // sectorsize
-    if sectors < (minsectors + 65536):
-        raise Exception('Disk too small to fit image')
-    biggestsectors = sectors - (minexcsize // sectorsize)
-    biggestsize = sectorsize * biggestsectors
-    parted.run('mklabel gpt')
-    curroffset = 2048
-    for volidx in sorted(plainvols):
-        vol = plainvols[volidx]
-        if vol is not biggestfs:
-            size = vol['initsize'] // sectorsize
-        else:
-            size = biggestsize // sectorsize
-        size += 2047 - (size % 2048)
-        end = curroffset + size
-        if end > sectors:
-            end = sectors
-        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
-        curroffset += size + 1
-    if not lvmvols:
-        if swapsize:
-            swapsize = swapsize // sectorsize
-            swapsize += 2047 - (size % 2048)
-            end = curroffset + swapsize
-            if end > sectors:
-                end = sectors
-            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
-    else:
-        parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
-        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
-        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
-        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
-        vginfo = vginfo.split('\n')
-        pesize = 0
-        pes = 0
-        for infline in vginfo:
-            infline = infline.split()
-            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
-                pesize = int(infline[2])
-            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
-                pes = int(infline[4])
-        takeaway = swapsize // pesize
-        for volidx in lvmvols:
-            vol = lvmvols[volidx]
-            if vol is biggestfs:
-                continue
-            takeaway += vol['initsize'] // pesize
-            takeaway += 1
-        biggestextents = pes - takeaway
-        for volidx in lvmvols:
-            vol = lvmvols[volidx]
-            if vol is biggestfs:
-                extents = biggestextents
-            else:
-                extents = vol['initsize'] // pesize
-                extents += 1
-            if vol['mount'] == '/':
-                lvname = 'root'
-            else:
-                lvname = vol['mount'].replace('/', '_')
-            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
-            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
-        if swapsize:
-            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
-            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
-        os.makedirs('/run/imginst/targ')
-        for vol in allvols:
-            with open(vol['targetdisk'], 'wb') as partition:
-                partition.write(b'\x00' * 1 * 1024 * 1024)
-            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
-            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
-            source = vol['mount'].replace('/', '_')
-            source = '/run/imginst/sources/' + source
-            blankfsstat = os.statvfs('/run/imginst/targ')
-            blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
-            sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
-            with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier:
-                stillrunning = copier.poll()
-                lastprogress = 0.0
-                while stillrunning is None:
-                    currfsstat = os.statvfs('/run/imginst/targ')
-                    currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize
-                    currused -= blankused
-                    with open('/proc/meminfo') as meminf:
-                        for line in meminf.read().split('\n'):
-                            if line.startswith('Dirty:'):
-                                _, dirty, _ = line.split()
-                                dirty = int(dirty) * 1024
-                    progress = (currused - dirty) / vol['minsize']
-                    if progress < lastprogress:
-                        progress = lastprogress
-                    if progress > 0.99:
-                        progress = 0.99
-                    lastprogress = progress
-                    progress = progress * 100
-                    sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
-                    sys.stdout.flush()
-                    time.sleep(0.5)
-                    stillrunning = copier.poll()
-                if stillrunning != 0:
-                    raise Exception("Error copying volume")
-                with subprocess.Popen(['sync']) as syncrun:
-                    stillrunning = syncrun.poll()
-                    while stillrunning is None:
-                        with open('/proc/meminfo') as meminf:
-                            for line in meminf.read().split('\n'):
-                                if line.startswith('Dirty:'):
-                                    _, dirty, _ = line.split()
-                                    dirty = int(dirty) * 1024
-                        progress = (vol['minsize'] - dirty) / vol['minsize']
-                        if progress < lastprogress:
-                            progress = lastprogress
-                        if progress > 0.99:
-                            progress = 0.99
-                        lastprogress = progress
-                        progress = progress * 100
-                        sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
-                        sys.stdout.flush()
-                        time.sleep(0.5)
-                        stillrunning = syncrun.poll()
-                sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
-                sys.stdout.write('\n')
-                sys.stdout.flush()
-            subprocess.check_call(['umount', '/run/imginst/targ'])
-        for vol in allvols:
-            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
-        fixup('/run/imginst/targ', allvols)
-
-
-if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -1,121 +0,0 @@
-. /lib/dracut-lib.sh
-mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
-    mount -t tmpfs untethered /mnt/remoteimg
-    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
-else
-    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
-    /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
-fi
-/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
-loopdev=$(losetup -f)
-export mountsrc=$loopdev
-losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
-if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
-    curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key
-    cipher=$(head -n 1 /tmp/rootimg.key)
-    key=$(tail -n 1 /tmp/rootimg.key)
-    len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
-    len=$(((len-4096)/512))
-    dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
-    /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
-    mountsrc=/dev/mapper/cryptimg
-fi
-
-if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then
-    mount -o ro $mountsrc /mnt/remote
-elif grep  '^Format: confluent_multisquash' /tmp/rootimg.info; then
-    tail -n +3 /tmp/rootimg.info  | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
-    . /tmp/setupmount.sh
-    cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
-    . /tmp/mountparts.sh
-fi
-
-
-#mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /mnt/overlay/upper /mnt/overlay/work
-    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
-else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
-        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
-        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
-    done
-fi
-mkdir -p /sysroot/etc/ssh
-mkdir -p /sysroot/etc/confluent
-mkdir -p /sysroot/root/.ssh
-cp /root/.ssh/* /sysroot/root/.ssh
-chmod 700 /sysroot/root/.ssh
-cp /etc/confluent/* /sysroot/etc/confluent/
-cp /etc/ssh/*key* /sysroot/etc/ssh/
-for pubkey in /etc/ssh/ssh_host*key.pub; do
-    certfile=${pubkey/.pub/-cert.pub}
-    privfile=${pubkey%.pub}
-    if [ -s $certfile ]; then
-        echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
-    fi
-    echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
-done
-
-mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
-if [ ! -z "$autocons" ]; then
-    autocons=${autocons%,*}
-    mkdir -p /run/systemd/generator/getty.target.wants
-    ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
-fi
-while [ ! -e /sysroot/sbin/init ]; do
-    echo "Failed to access root filesystem or it is missing /sbin/init"
-    echo "System should be accessible through ssh at port 2222 with the appropriate key"
-    while [ ! -e /sysroot/sbin/init ]; do
-        sleep 1
-    done
-done
-rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
-rootpassword=${rootpassword#rootpassword: }
-if [ "$rootpassword" = "null" ]; then
-    rootpassword=""
-fi
-
-if [ ! -z "$rootpassword" ]; then
-    sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
-fi
-for i in /ssh/*.ca; do
-    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
-done
-echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
-echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
-echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
-sshconf=/sysroot/etc/ssh/ssh_config
-if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
-    sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
-fi
-echo 'Host *' >> $sshconf
-echo '    HostbasedAuthentication yes' >> $sshconf
-echo '    EnableSSHKeysign yes' >> $sshconf
-echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
-cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
-chmod 640 /sysroot/etc/ssh/*_key
-chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
-cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
-chroot /sysroot/ update-ca-trust
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
-mkdir -p /sysroot/opt/confluent/bin
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
-chmod +x /sysroot/opt/confluent/bin/onboot.sh
-cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
-ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
-cp /etc/confluent/functions /sysroot/etc/confluent/functions
-if grep installtodisk /proc/cmdline > /dev/null; then
-    . /etc/confluent/functions
-    run_remote installimage
-    exec reboot -f
-fi
-exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -1,43 +0,0 @@
-#!/bin/bash
-. /etc/confluent/functions
-# the image will be used to deploy itself
-# provide both access to image (for parsing metadata)
-# and existing mounts of image (to take advantage of caching)
-mount -o bind /sys /sysroot/sys
-mount -o bind /dev /sysroot/dev
-mount -o bind /proc /sysroot/proc
-mount -o bind /run /sysroot/run
-
-
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /sysroot/run/imginst/sources/_
-    mount -o bind /mnt/remote /sysroot/run/imginst/sources/_
-else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do
-        srcname=${srcmount#/dev/mapper/mproot}
-        srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g')
-        mkdir -p /sysroot/run/imginst/sources/$srcname
-        mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname
-    done
-fi
-cd /sysroot/run
-chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
-chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
-if [ ! -f /sysroot/tmp/installdisk ]; then
-    echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
-    while [ ! -f /sysroot/tmp/installdisk ]; do
-        sleep 1
-    done
-fi
-lvm vgchange -a n
-udevadm control -e
-chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
-chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
-echo "Port 22" >> /etc/ssh/sshd_config
-echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
-echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
-kill -HUP $(cat /run/sshd.pid)
-
-chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
-chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"
-

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Confluent onboot hook
-Requires=network-online.target
-After=network-online.target
-
-[Service]
-ExecStart=/opt/confluent/bin/onboot.sh
-
-[Install]
-WantedBy=multi-user.target
-

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-# This script is executed on each boot as it is
-# completed. It is best to edit the middle of the file as
-# noted below so custom commands are executed before
-# the script notifies confluent that install is fully complete.
-
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
-export nodename confluent_mgr confluent_profile
-. /etc/confluent/functions
-mkdir -p /var/log/confluent
-exec >> /var/log/confluent/confluent-onboot.log
-exec 2>> /var/log/confluent/confluent-onboot.log
-tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
-logshowpid=$!
-
-run_remote_python syncfileclient
-
-run_remote onboot.custom
-# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
-run_remote_parts onboot.d
-
-# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
-run_remote_config onboot.d
-
-#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
-kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/post.sh

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-# This script is executed 'chrooted' into a cloned disk target before rebooting
-#
-
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
-export nodename confluent_mgr confluent_profile
-. /etc/confluent/functions
-mkdir -p /var/log/confluent
-exec >> /var/log/confluent/confluent-post.log
-exec 2>> /var/log/confluent/confluent-post.log
-tail -f /var/log/confluent/confluent-post.log > /dev/console &
-logshowpid=$!
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
-mkdir -p /opt/confluent/bin
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
-chmod +x /opt/confluent/bin/firstboot.sh
-systemctl enable firstboot
-selinuxpolicy=$(grep ^SELINUXTYPE /etc/selinux/config |awk -F= '{print $2}')
-if [ ! -z "$selinuxpolicy" ]; then
-    setfiles /etc/selinux/${selinuxpolicy}/contexts/files/file_contexts /etc/
-fi
-run_remote_python syncfileclient
-run_remote post.custom
-# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh
-run_remote_parts post.d
-
-# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
-run_remote_config post.d
-
-curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
-
-kill $logshowpid
-

confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient

@@ -1,219 +0,0 @@
-#!/usr/bin/python
-import importlib
-import tempfile
-import json
-import os
-import shutil
-from importlib.machinery import SourceFileLoader
-try:
-    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
-except FileNotFoundError:
-    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
-
-
-def partitionhostsline(line):
-    comment = ''
-    try:
-        cmdidx = line.index('#')
-        comment = line[cmdidx:]
-        line = line[:cmdidx].strip()
-    except ValueError:
-        pass
-    if not line:
-        return '', [], comment
-    ipaddr, names = line.split(maxsplit=1)
-    names = names.split()
-    return ipaddr, names, comment
-
-class HostMerger:
-    def __init__(self):
-        self.byip = {}
-        self.byname = {}
-        self.sourcelines = []
-        self.targlines = []
-
-    def read_source(self, sourcefile):
-        with open(sourcefile, 'r') as hfile:
-            self.sourcelines = hfile.read().split('\n')
-        while not self.sourcelines[-1]:
-            self.sourcelines = self.sourcelines[:-1]
-        for x in range(len(self.sourcelines)):
-            line = self.sourcelines[x]
-            currip, names, comment = partitionhostsline(line)
-            if currip:
-                self.byip[currip] = x
-            for name in names:
-                self.byname[name] = x
-
-    def read_target(self, targetfile):
-        with open(targetfile, 'r') as hfile:
-            lines = hfile.read().split('\n')
-            if not lines[-1]:
-                lines = lines[:-1]
-            for y in range(len(lines)):
-                line = lines[y]
-                currip, names, comment = partitionhostsline(line)
-                if currip in self.byip:
-                    x = self.byip[currip]
-                    if self.sourcelines[x] is None:
-                        # have already consumed this enntry
-                        continue
-                    self.targlines.append(self.sourcelines[x])
-                    self.sourcelines[x] = None
-                    continue
-                for name in names:
-                    if name in self.byname:
-                        x = self.byname[name]
-                        if self.sourcelines[x] is None:
-                            break
-                        self.targlines.append(self.sourcelines[x])
-                        self.sourcelines[x] = None
-                        break
-                else:
-                    self.targlines.append(line)
-
-    def write_out(self, targetfile):
-        while not self.targlines[-1]:
-            self.targlines = self.targlines[:-1]
-        while not self.sourcelines[-1]:
-            self.sourcelines = self.sourcelines[:-1]
-        with open(targetfile, 'w') as hosts:
-            for line in self.targlines:
-                hosts.write(line + '\n')
-            for line in self.sourcelines:
-                if line is not None:
-                    hosts.write(line + '\n')
-
-
-class CredMerger:
-    def __init__(self):
-        try:
-            with open('/etc/login.defs', 'r') as ldefs:
-                defs = ldefs.read().split('\n')
-        except FileNotFoundError:
-            defs = []
-        lkup = {}
-        self.discardnames = {}
-        self.shadowednames = {}
-        for line in defs:
-            try:
-                line = line[:line.index('#')]
-            except ValueError:
-                pass
-            keyval = line.split()
-            if len(keyval) < 2:
-                continue
-            lkup[keyval[0]] = keyval[1]
-        self.uidmin = int(lkup.get('UID_MIN', 1000))
-        self.uidmax = int(lkup.get('UID_MAX', 60000))
-        self.gidmin = int(lkup.get('GID_MIN', 1000))
-        self.gidmax = int(lkup.get('GID_MAX', 60000))
-        self.shadowlines = None
-
-    def read_passwd(self, source, targfile=False):
-        self.read_generic(source, self.uidmin, self.uidmax, targfile)
-
-    def read_group(self, source, targfile=False):
-        self.read_generic(source, self.gidmin, self.gidmax, targfile)
-
-    def read_generic(self, source, minid, maxid, targfile):
-        if targfile:
-            self.targdata = []
-        else:
-            self.sourcedata = []
-        with open(source, 'r') as inputfile:
-            for line in inputfile.read().split('\n'):
-                try:
-                    name, _, uid, _ = line.split(':', 3)
-                    uid = int(uid)
-                except ValueError:
-                    continue
-                if targfile:
-                    if uid <= minid or uid >= maxid:
-                        self.targdata.append(line)
-                    else:
-                        self.discardnames[name] = 1
-                else:
-                    if uid >= minid and uid <= maxid:
-                        self.sourcedata.append(line)
-
-    def read_shadow(self, source):
-        self.shadowlines = []
-        try:
-            with open(source, 'r') as inshadow:
-                for line in inshadow.read().split('\n'):
-                    try:
-                        name, _ = line.split(':' , 1)
-                    except ValueError:
-                        continue
-                    if name in self.discardnames:
-                        continue
-                    self.shadowednames[name] = 1
-                    self.shadowlines.append(line)
-        except FileNotFoundError:
-            return
-
-    def write_out(self, outfile):
-        with open(outfile, 'w') as targ:
-            for line in self.targdata:
-                targ.write(line + '\n')
-            for line in self.sourcedata:
-                targ.write(line + '\n')
-        if outfile == '/etc/passwd':
-            if self.shadowlines is None:
-                self.read_shadow('/etc/shadow')
-            with open('/etc/shadow', 'w') as shadout:
-                for line in self.shadowlines:
-                    shadout.write(line + '\n')
-                for line in self.sourcedata:
-                    name, _ = line.split(':', 1)
-                    if name in self.shadowednames:
-                        continue
-                    shadout.write(name + ':!:::::::\n')
-        if outfile == '/etc/group':
-            if self.shadowlines is None:
-                self.read_shadow('/etc/gshadow')
-            with open('/etc/gshadow', 'w') as shadout:
-                for line in self.shadowlines:
-                    shadout.write(line + '\n')
-                for line in self.sourcedata:
-                    name, _ = line.split(':' , 1)
-                    if name in self.shadowednames:
-                        continue
-                    shadout.write(name + ':!::\n')
-
-def synchronize():
-    tmpdir = tempfile.mkdtemp()
-    try:
-        ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir})
-        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
-        if status == 202:
-            while status != 204:
-                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
-                if not isinstance(rsp, str):
-                    rsp = rsp.decode('utf8')
-            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
-            if os.path.exists(pendpasswd):
-                cm = CredMerger()
-                cm.read_passwd(pendpasswd, targfile=False)
-                cm.read_passwd('/etc/passwd', targfile=True)
-                cm.write_out('/etc/passwd')
-            pendgroup = os.path.join(tmpdir, 'etc/group')
-            if os.path.exists(pendgroup):
-                cm = CredMerger()
-                cm.read_group(pendgroup, targfile=False)
-                cm.read_group('/etc/group', targfile=True)
-                cm.write_out('/etc/group')
-            pendhosts = os.path.join(tmpdir, 'etc/hosts')
-            if os.path.exists(pendhosts):
-                cm = HostMerger()
-                cm.read_source(pendhosts)
-                cm.read_target('/etc/hosts')
-                cm.write_out('/etc/hosts')
-    finally:
-        shutil.rmtree(tmpdir)
-
-
-if __name__ == '__main__':
-    synchronize()

confluent_osdeploy/el8-diskless/profiles/default/syncfiles

@@ -1,13 +0,0 @@
-# This file lists files to synchronize or merge to the deployed systems from the deployment server
-# To specify taking /some/path/hosts on the deploymennt server and duplicating it to /etc/hosts:
-# /some/path/hosts -> /etc/hosts
-
-# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
-# /etc/hosts
-
-# Entries under MERGE will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
-# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
-# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
-MERGE:
-# /etc/passwd
-# /etc/group

confluent_osdeploy/el8/profiles/default/kickstart

@@ -35,13 +35,11 @@ reboot
 %packages
 @^minimal-environment
 #-kernel-uek # This can opt out of the UEK for the relevant distribution
-bind-utils
 chrony
-pciutils
-python3
 rsync
+python3
 tar
--iwl*-firmware
+pciutils
 %include /tmp/addonpackages
 %include /tmp/cryptpkglist
 %end

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -15,9 +15,6 @@ exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
 logshowpid=$!
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
-	sleep 1
-done
 
 if [ ! -f /etc/confluent/firstboot.ran ]; then
     touch /etc/confluent/firstboot.ran

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -1,7 +1,4 @@
 function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
     if [ -z "$confluent_mgr" ]; then
         confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
@@ -54,7 +51,7 @@ source_remote() {
     chmod +x $1
     cmd=$1
     shift
-    source ./$cmd
+    source $cmd
     cd - > /dev/null
     return $retcode
 }

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -46,11 +46,6 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
-        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
-            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
-                self.size = int(sizesrc.read()) * 512
-        if int(self.size) < 536870912:
-            raise Exception("Device too small for install")
 
     @property
     def priority(self):

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.post

@@ -14,6 +14,6 @@ if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
     tar xf infiniband/mofed.tgz
     # The rest is common between tar and iso
     cd MLNX_OFED*
-    ./mlnxofedinstall --force --without-32bit
+    ./mlnxofedinstall --force
 fi
 

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.pre

@@ -1,11 +1,10 @@
 # Add needed base packages to the install
 cat << EOF >> /tmp/addonpackages
-gcc-gfortran
-kernel-modules-extra
-lsof
 perl
 pkgconf-pkg-config
-tcl
 tcsh
+lsof
 tk
+gcc-gfortran
+tcl
 EOF

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -76,14 +76,6 @@ fi
 export confluent_mgr confluent_profile nodename
 curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /tmp/functions
 . /tmp/functions
-if [ -e /usr/libexec/platform-python ]; then
-    python=/usr/libexec/platform-python
-elif [ -e /usr/bin/python3 ]; then
-    python=/usr/byn/python3
-else
-    python=/usr/bin/python
-fi
-$python /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
 run_remote pre.custom
 run_remote_parts pre.d
 if [ ! -e /tmp/installdisk ]; then
@@ -94,4 +86,12 @@ if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then
     echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
     echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
 fi
+if [ -e /usr/libexec/platform-python ]; then
+    python=/usr/libexec/platform-python
+elif [ -e /usr/bin/python3 ]; then
+    python=/usr/byn/python3
+else
+    python=/usr/bin/python
+fi
+$python /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
 kill $logshowpid

confluent_osdeploy/el8/profiles/default/scripts/syncfileclient

@@ -134,9 +134,7 @@ class CredMerger:
                     else:
                         self.discardnames[name] = 1
                 else:
-                    if name[0] in ('+', '#', '@'):
-                        self.sourcedata.append(line)
-                    elif uid >= minid and uid <= maxid:
+                    if uid >= minid and uid <= maxid:
                         self.sourcedata.append(line)
 
     def read_shadow(self, source):
@@ -169,8 +167,6 @@ class CredMerger:
                     shadout.write(line + '\n')
                 for line in self.sourcedata:
                     name, _ = line.split(':', 1)
-                    if name[0] in ('+', '#', '@'):
-                        continue
                     if name in self.shadowednames:
                         continue
                     shadout.write(name + ':!:::::::\n')
@@ -220,4 +216,4 @@ def synchronize():
 
 
 if __name__ == '__main__':
-    synchronize()
+    synchronize()

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -8,13 +8,13 @@ if ! grep console= /proc/cmdline >& /dev/null; then
     echo $autocons > /tmp/01-autocons.devnode
     if [ ! -z "$autocons" ]; then
         echo "Using $(cat /tmp/01-autocons.conf)"
-        (while :; do TERM=xterm-256color tmux a <> $autocons >&0 2>&1; done) &
+        (while :; do tmux a <> $autocons >&0 2>&1; done) &
         (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
     else
-        (while :; do TERM=linux tmux a <> /dev/console >&0 2>&1; done) &
+        (while :; do tmux a <> /dev/console >&0 2>&1; done) &
     fi
 else
-    (while :; do TERM=xterm-256color tmux a <> /dev/console >&0 2>&1; done) &
+    (while :; do tmux a <> /dev/console >&0 2>&1; done) &
     (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
 fi
 (while :; do TERM=linux tmux <> /dev/tty2 >&0 2>&1; done) &
@@ -55,74 +55,17 @@ done
 cd /
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 hostname $nodename
-tpmdir=$(mktemp -d)
-cd $tpmdir
-lasthdl=""
-oldumask=$(umask)
-umask 0077
-for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
-    tpm2_startauthsession --policy-session --session=session.ctx
-    tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
-    unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
-    tpm2_flushcontext session.ctx
-    if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
-        confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
-        echo $confluent_apikey > /etc/confluent/confluent.apikey
-        if [ -n "$lasthdl" ]; then
-            tpm2_evictcontrol -c $lasthdl
-        fi
-        lasthdl=$hdl
-    fi
-done
-cd - > /dev/null
-rm -rf $tpmdir
 /usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-umask $oldumask
-new_apikey=$(cat /etc/confluent/confluent.apikey)
-if [ "$new_apikey" != "$confluent_apikey" ]; then
-    if [ -n "$lasthdl" ]; then
-        tpm2_evictcontrol -c $lasthdl
-    fi
-    tmpdir=$(mktemp -d)
-    cd $tmpdir
-    tpm2_startauthsession --session=session.ctx
-    tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
-    tpm2_createprimary -G ecc -Q --key-context=prim.ctx
-    (echo -n "CONFLUENT_APIKEY:$new_apikey") | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
-    tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
-    tpm2_evictcontrol -Q -c data.ctx
-    tpm2_flushcontext session.ctx
-    cd - > /dev/null
-    rm -rf $tmpdir
-fi
-tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2 2> /dev/null
 ifidx=$(cat /tmp/confluent.ifidx)
 ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
 ifname=${ifname%:}
-autoconfigmethod=$(grep ipv6_method /etc/confluent/confluent.deploycfg |awk '{print $2}')
-if [ "$autoconfigmethod" = "static" ]; then
-    ipaddr=$(grep ^ipv6_address: /etc/confluent/confluent.deploycfg)
-    ipaddr=${ipaddr#ipv6_address: }
-    ipgw=$(grep ^ipv6_gateway: /etc/confluent/confluent.deploycfg)
-    ipgw=${ipgw#ipv6_gateway: }
-    if [ "$ipgw" = "null" ]; then
-        ipgw=""
-    fi
-    ipnm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
-    ipnm=${ipnm#prefix: }
-    echo "Setting up $ifname as static at $ipaddr/$ipnm"
-    ip addr add dev $ifname $ipaddr/$ipnm
-    if [ ! -z "$ipgw" ]; then
-        ip route add default via $ipgw
-    fi
-fi
 autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg |awk '{print $2}')
 if [ "$autoconfigmethod" = "dhcp" ]; then
     echo -n "Attempting to use dhcp to bring up $ifname..."
     dhclient $ifname
     echo "Complete:"
     ip addr show dev $ifname
-elif [ "$autocinfigmethtod" = "static" ]; then
+else
     v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
     v4addr=${v4addr#ipv4_address: }
     v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)

confluent_osdeploy/genesis/profiles/default/scripts/functions

@@ -1,7 +1,4 @@
 function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
     if [ -z "$confluent_mgr" ]; then
         confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi

confluent_osdeploy/rhvh4/profiles/default/scripts/firstboot.sh

@@ -12,9 +12,6 @@ profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
 export nodename mgr profile
 . /etc/confluent/functions
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
-	sleep 1
-done
 
 
 run_remote firstboot.custom

confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -1,193 +0,0 @@
-get_remote_apikey() {
-    while [ -z "$confluent_apikey" ]; do
-        /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
-        if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
-            # we don't support remote sealed api keys
-            echo > /etc/confluent/confluent.apikey
-        fi
-        confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-        if [ -z "$confluent_apikey" ]; then
-            echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
-            sleep 10
-        else
-            tmpdir=$(mktemp -d)
-            cd $tmpdir
-            tpm2_startauthsession --session=session.ctx
-            tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
-            tpm2_createprimary -G ecc -Q --key-context=prim.ctx
-            (echo -n "CONFLUENT_APIKEY:";cat /etc/confluent/confluent.apikey) | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
-            tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
-            tpm2_evictcontrol -Q -c data.ctx
-            tpm2_flushcontext session.ctx
-            cd - > /dev/null
-            rm -rf $tmpdir
-        fi
-    done
-}
-root=1
-rootok=1
-netroot=confluent
-clear
-mkdir -p /etc/ssh
-mkdir -p /var/tmp/
-mkdir -p /var/lib/empty
-mkdir -p /var/empty/sshd
-mkdir -p /etc/confluent
-sed -i '/^root:x/d' /etc/passwd
-echo root:x:0:0::/:/bin/bash >> /etc/passwd
-echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
-
-if ! grep console= /proc/cmdline >& /dev/null; then
-    autocons=$(/opt/confluent/bin/autocons)
-    autocons=${autocons##*/}
-    echo "Automatic console configured for $autocons"
-fi
-echo "Initializing confluent diskless environment"
-echo -n "udevd: "
-/usr/lib/systemd/systemd-udevd --daemon
-echo -n "Loading drivers..."
-udevadm trigger
-udevadm trigger --type=devices --action=add
-udevadm settle
-modprobe ib_ipoib
-modprobe ib_umad
-modprobe hfi1
-modprobe mlx5_ib
-echo "done"
-cat > /etc/ssh/sshd_config << EOF
-Port 2222
-Subsystem       sftp    /usr/libexec/openssh/sftp-server
-PermitRootLogin yes
-AuthorizedKeysFile      .ssh/authorized_keys
-EOF
-mkdir /root/.ssh
-mkdir /.ssh
-cat /ssh/*pubkey > /root/.ssh/authorized_keys 2>/dev/null
-cp /root/.ssh/authorized_keys /.ssh/
-cat /tls/*.pem > /etc/confluent/ca.pem
-mkdir -p /var/lib/ca-certificates/
-cat /tls/*.pem > /var/lib/ca-certificates/ca-bundle.pem
-TRIES=0
-oldumask=$(umask)
-umask 0077
-tpmdir=$(mktemp -d)
-cd $tpmdir
-lasthdl=""
-for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
-    tpm2_startauthsession --policy-session --session=session.ctx
-    tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
-    unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
-    tpm2_flushcontext session.ctx
-    if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
-        confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
-        echo $confluent_apikey > /etc/confluent/confluent.apikey
-        if [ -n "$lasthdl" ]; then
-            tpm2_evictcontrol -c $lasthdl
-        fi
-        lasthdl=$hdl
-    fi
-done
-cd - > /dev/null
-rm -rf $tpmdir
-touch /etc/confluent/confluent.info
-cd /sys/class/net
-echo -n "Scanning for network configuration..."
-while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
-    TRIES=$((TRIES + 1))
-    for i in *; do
-        ip link set $i up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
-done
-cd /
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-hostname $nodename
-confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1|awk -F': ' '{print $2}' | awk -F'|' '{print $1}')
-if [ -z "$confluent_mgr" ]; then
-    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
-fi
-if [[ $confluent_mgr == *%* ]]; then
-    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
-    ifidx=$(cat /tmp/confluent.ifidx)
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-fi
-
-ready=0
-while [ $ready = "0" ]; do
-    get_remote_apikey
-    if [[ $confluent_mgr == *:* ]]; then
-        confluent_mgr="[$confluent_mgr]"
-    fi
-    tmperr=$(mktemp)
-    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
-    if grep 401 $tmperr > /dev/null; then
-        confluent_apikey=""
-        if [ -n "$lasthdl" ]; then
-            tpm2_evictcontrol -c $lasthdl
-        fi
-        confluent_mgr=${confluent_mgr#[}
-        confluent_mgr=${confluent_mgr%]}
-    else
-        ready=1
-    fi
-    rm $tmperr
-done
-tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
-umask $oldumask
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg |awk '{print $2}')
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo -n "Attempting to use dhcp to bring up $ifname..."
-    dhclient $ifname
-    echo "Complete:"
-    ip addr show dev $ifname
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
-    fi
-    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#prefix: }
-    echo "Setting up $ifname as static at $v4addr/$v4nm"
-    ip addr add dev $ifname $v4addr/$v4nm
-    if [ ! -z "$v4gw" ]; then
-        ip route add default via $v4gw
-    fi
-    mkdir -p /run/confluent
-    echo -e "BOOTPROTO='static'\nSTARTMODE='auto'" >> /run/confluent/ifcfg-$ifname
-    echo "IPADDR='$v4addr/$v4nm'" >> /run/confluent/ifcfg-$ifname
-    if [ ! -z "$v4gw" ]; then
-        echo defafult $v4gw - $ifname > /run/confluent/ifroute-$ifname
-    fi
-fi
-
-echo -n "Initializing ssh..."
-ssh-keygen -A
-for pubkey in /etc/ssh/ssh_host*key.pub; do
-    certfile=${pubkey/.pub/-cert.pub}
-    privfile=${pubkey%.pub}
-    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
-    if [ -s $certfile ]; then
-        echo HostCertificate $certfile >> /etc/ssh/sshd_config
-    fi
-    echo HostKey $privfile >> /etc/ssh/sshd_config
-done
-/usr/sbin/sshd
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
-confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
-confluent_urls=""
-for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed -e s/%/%25/); do
-    if [[ $addr == *:* ]]; then
-        confluent_urls="$confluent_urls $confluent_proto://[$addr]/confluent-public/os/$confluent_profile/rootimg.sfs"
-    else
-        confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
-    fi
-done
-confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
-mkdir -p /etc/confluent
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
-. /etc/confluent/functions
-source_remote imageboot.sh

confluent_osdeploy/suse15-diskless/profiles/default/scripts/functions

@@ -1,144 +0,0 @@
-function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
-    if [ -z "$confluent_mgr" ]; then
-        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
-    fi
-    if [ -z "$confluent_profile" ]; then
-        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
-    fi
-}
-
-fetch_remote() {
-    curlargs=""
-    if [ -f /etc/confluent/ca.pem ]; then
-        curlargs=" --cacert /etc/confluent/ca.pem"
-    fi
-    set_confluent_vars
-    mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
-    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
-}
-
-source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d)
-    apiclient=/opt/confluent/bin/apiclient
-    if [ -f /etc/confluent/apiclient ]; then
-        apiclient=/etc/confluent/apiclient
-    fi
-    scriptlist=$(/usr/libexec/platform-python $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
-    for script in $scriptlist; do
-        source_remote $1/$script
-    done
-    unset confluentscripttmpdir
-}
-
-run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d)
-    apiclient=/opt/confluent/bin/apiclient
-    if [ -f /etc/confluent/apiclient ]; then
-        apiclient=/etc/confluent/apiclient
-    fi
-    scriptlist=$(/usr/libexec/platform-python $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
-    for script in $scriptlist; do
-        run_remote $1/$script
-    done
-    unset confluentscripttmpdir
-}
-
-source_remote() {
-    set_confluent_vars
-    echo
-    echo '---------------------------------------------------------------------------'
-    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d)
-    fi
-    echo Sourcing from $confluentscripttmpdir
-    cd $confluentscripttmpdir
-    fetch_remote $1
-    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
-    chmod +x $1
-    cmd=$1
-    shift
-    source ./$cmd
-    cd - > /dev/null
-    return $retcode
-}
-
-run_remote() {
-    requestedcmd="'$*'"
-    set_confluent_vars
-    echo
-    echo '---------------------------------------------------------------------------'
-    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d)
-    fi
-    echo Executing in $confluentscripttmpdir
-    cd $confluentscripttmpdir
-    fetch_remote $1
-    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
-    chmod +x $1
-    cmd=$1
-    if [ -x /usr/bin/chcon ]; then
-        chcon system_u:object_r:bin_t:s0 $cmd
-    fi
-    shift
-    ./$cmd $*
-    retcode=$?
-    if [ $retcode -ne 0 ]; then
-        echo "$requestedcmd exited with code $retcode"
-    fi
-    cd - > /dev/null
-    return $retcode
-}
-
-run_remote_python() {
-    echo
-    set_confluent_vars
-    if [ -f /etc/confluent/ca.pem ]; then
-        curlargs=" --cacert /etc/confluent/ca.pem"
-    fi
-    echo '---------------------------------------------------------------------------'
-    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    tmpdir=$(mktemp -d)
-    echo Executing in $tmpdir
-    cd $tmpdir
-    mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
-    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
-    if [ -x /usr/libexec/platform-python ]; then
-        /usr/libexec/platform-python $*
-    else
-        /usr/bin/python $*
-    fi
-    retcode=$?
-    echo "'$*' exited with code $retcode"
-    cd - > /dev/null
-    return $retcode
-}
-
-run_remote_config() {
-    echo
-    set_confluent_vars
-    apiclient=/opt/confluent/bin/apiclient
-    if [ -f /etc/confluent/apiclient ]; then
-        apiclient=/etc/confluent/apiclient
-    fi
-    echo '---------------------------------------------------------------------------'
-    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
-    if [ -x /usr/libexec/platform-python ]; then
-        /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
-        /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/status -w 204
-    else
-        /usr/bin/python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
-        /usr/bin/python $apiclient /confluent-api/self/remoteconfig/status -w 204
-    fi
-    echo
-    echo 'Completed remote configuration'
-    echo '---------------------------------------------------------------------------'
-    return
-}
-

confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh

@@ -1,139 +0,0 @@
-. /lib/dracut-lib.sh
-mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
-if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
-    mount -t tmpfs untethered /mnt/remoteimg
-    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
-else
-    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
-    /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
-fi
-/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
-loopdev=$(losetup -f)
-export mountsrc=$loopdev
-losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
-if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
-    curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key
-    cipher=$(head -n 1 /tmp/rootimg.key)
-    key=$(tail -n 1 /tmp/rootimg.key)
-    len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
-    len=$(((len-4096)/512))
-    dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
-    /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
-    mountsrc=/dev/mapper/cryptimg
-fi
-if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then
-    mount -o ro $mountsrc /mnt/remote
-elif grep  '^Format: confluent_multisquash' /tmp/rootimg.info; then
-    tail -n +3 /tmp/rootimg.info  | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
-    . /tmp/setupmount.sh
-    cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
-    . /tmp/mountparts.sh
-fi
-
-#mount -t tmpfs overlay /mnt/overlay
-modprobe zram
-memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
-memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
-echo $memtot > /sys/block/zram0/disksize
-mkfs.xfs /dev/zram0 > /dev/null
-mount -o discard /dev/zram0 /mnt/overlay
-if [ ! -f /tmp/mountparts.sh ]; then
-    mkdir -p /mnt/overlay/upper /mnt/overlay/work
-    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
-else
-    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
-        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
-        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
-    done
-fi
-mkdir -p /sysroot/etc/ssh
-mkdir -p /sysroot/etc/confluent
-mkdir -p /sysroot/root/.ssh
-cp /root/.ssh/* /sysroot/root/.ssh
-chmod 700 /sysroot/root/.ssh
-cp /etc/confluent/* /sysroot/etc/confluent/
-cp /etc/ssh/*key* /sysroot/etc/ssh/
-for pubkey in /etc/ssh/ssh_host*key.pub; do
-    certfile=${pubkey/.pub/-cert.pub}
-    privfile=${pubkey%.pub}
-    if [ -s $certfile ]; then
-        echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
-    fi
-    echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
-done
-
-mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
-if [ ! -z "$autocons" ]; then
-    autocons=${autocons%,*}
-    mkdir -p /run/systemd/generator/getty.target.wants
-    ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
-fi
-while [ ! -e /sysroot/sbin/init ]; do
-    echo "Failed to access root filesystem or it is missing /sbin/init"
-    echo "System should be accessible through ssh at port 2222 with the appropriate key"
-    while [ ! -e /sysroot/sbin/init ]; do
-        sleep 1
-    done
-done
-rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
-rootpassword=${rootpassword#rootpassword: }
-if [ "$rootpassword" = "null" ]; then
-    rootpassword=""
-fi
-
-if [ ! -z "$rootpassword" ]; then
-    sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
-fi
-for i in /ssh/*.ca; do
-    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
-done
-echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
-echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
-echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
-sshconf=/sysroot/etc/ssh/ssh_config
-if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
-    sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
-fi
-echo 'Host *' >> $sshconf
-echo '    HostbasedAuthentication yes' >> $sshconf
-echo '    EnableSSHKeysign yes' >> $sshconf
-echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
-cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
-chmod 600 /sysroot/etc/ssh/*_key
-chroot /sysroot cat /etc/confluent/ca.pem >> /sysroot/var/lib/ca-certificates/ca-bundle.pem
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
-mkdir -p /sysroot/opt/confluent/bin
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
-chmod +x /sysroot/opt/confluent/bin/onboot.sh
-cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
-ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
-cp /etc/confluent/functions /sysroot/etc/confluent/functions
-
-nameserversec=0
-nameservers=""
-while read -r entry; do
-    if [ $nameserversec = 1 ]; then
-        if [[ $entry == "-"* ]]; then
-            nameservers="$nameservers"${entry#- }" "
-            continue
-        fi
-    fi
-    nameserversec=0
-    if [ "${entry%:*}" = "nameservers" ]; then
-        nameserversec=1
-        continue
-    fi
-done < /etc/confluent/confluent.deploycfg
-nameservers=${nameservers% }
-sed -i 's/^NETCONFIG_DNS_STATIC_SERVERS="/NETCONFIG_DNS_STATIC_SERVERS="'"$nameservers"/ /sysroot/etc/sysconfig/network/config
-dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
-dnsdomain=${dnsdomain#dnsdomain: }
-sed -i 's/^NETCONFIG_DNS_STATIC_SEARCHLIST="/NETCONFIG_DNS_STATIC_SEARCHLIST="'$dnsdomain/ /sysroot/etc/sysconfig/network/config
-cp /run/confluent/ifroute-* /run/confluent/ifcfg-* /sysroot/etc/sysconfig/network
-if grep installtodisk /proc/cmdline > /dev/null; then
-    . /etc/confluent/functions
-    run_remote installimage
-    exec reboot -f
-fi
-exec /opt/confluent/bin/start_root

confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Confluent onboot hook
-Requires=network-online.target
-After=network-online.target
-
-[Service]
-ExecStart=/opt/confluent/bin/onboot.sh
-
-[Install]
-WantedBy=multi-user.target
-

confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh

@@ -1,28 +0,0 @@
-#!/bin/sh
-
-# This script is executed on each boot as it is
-# completed. It is best to edit the middle of the file as
-# noted below so custom commands are executed before
-# the script notifies confluent that install is fully complete.
-
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
-confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
-export nodename confluent_mgr confluent_profile
-. /etc/confluent/functions
-mkdir -p /var/log/confluent
-exec >> /var/log/confluent/confluent-onboot.log
-exec 2>> /var/log/confluent/confluent-onboot.log
-tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
-logshowpid=$!
-
-run_remote onboot.custom
-# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
-run_remote_parts onboot.d
-
-# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
-run_remote_config onboot.d
-
-#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
-kill $logshowpid

confluent_osdeploy/suse15/initramfs/opt/confluent/bin/suseagent

@@ -66,7 +66,7 @@ if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null && [
 	echo "Serial console has been autodected and enabled read-only for install" > ${autocons%,*}
 	echo "It will be read-write after install" > ${autocons%,*}
 	echo "If a fully functional console is desired over serial, add console=${autocons#/dev/} " > ${autocons%,*}
-	echo "to kernelargs in the profile.yaml file of the profile and run 'osdeploy updateboot <profile>" > ${autocons%,*}
+	echo "to kerneralgs in the profile.yaml file of the profile and run 'osdeploy updateboot <profile>" > ${autocons%,*}
 	#echo "Serial console autodetected and enabled, will be available upon install completion" > ${autocons%,*}
 	#echo "The installer will run in text mode on the graphics console. When ssh is available," > ${autocons%,*}
 	#echo "install progress may be checked by using ssh to access and run the screendump command" > ${autocons%,*}

confluent_osdeploy/suse15/profiles/hpc/autoyast

@@ -1,34 +1,34 @@
 <?xml version="1.0"?>
-<!DOCTYPE profile>
+<!DOCTYPE profile SYSTEM "/usr/share/YaST2/include/autoinstall/profile.dtd">
 <!--
 This autoyast file will be processed by pre.sh before applying. See pre.sh for
 info on modifying its behavior, and also search and replace '%%' to remove
 dynamic behavior and replace with static configuration.
 --> 
-<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
-    <timezone>
-      <hwclock>UTC</hwclock>
-      <!--INSERT:/tmp/timezone-->
-    </timezone>
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns" xmlns:xi="http://www.w3.org/2001/XInclude">
     <general>
       <self_update config:type="boolean">false</self_update>
+      <clock>
+        <hwclock>UTC</hwclock>
+        <xi:include href="file:///tmp/timezone"/>
+      </clock>
       <mode>
 	      <confirm config:type="boolean">false</confirm>
       </mode>
     </general>
     %%IFSLE%%
     <add-on>
-      <add_on_products config:type="list">
-        <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
-	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-hpc</product><product_dir>/Module-HPC</product_dir></listentry>
-  	    <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-server-applications</product><product_dir>/Module-Server-Applications</product_dir></listentry>
-	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-containers</product><product_dir>/Module-Containers</product_dir></listentry>
-	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>SLE_HPC</product><product_dir>/Product-HPC</product_dir></listentry>
-	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
-      </add_on_products>
-    </add-on>
+        <add_on_products config:type="list">
+            <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-hpc</product><product_dir>/Module-HPC</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-server-applications</product><product_dir>/Module-Server-Applications</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-containers</product><product_dir>/Module-Containers</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>SLE_HPC</product><product_dir>/Product-HPC</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
+        </add_on_products>
+</add-on>
     %%ENDIFSLE%%
-    <!--INSERT:/tmp/bootloader.xml-->
+    <xi:include href="file:///tmp/bootloader.xml"/>
     <software>
        %%IFSLE%%
        <products config:type="list">
@@ -39,8 +39,6 @@ dynamic behavior and replace with static configuration.
         <pattern>base</pattern>
       </patterns>
       <packages config:type="list">
-	<package>openssh</package>
-	<package>iputils</package>
         <package>python3</package>
         <package>openssl</package>
         <package>chrony</package>
@@ -81,7 +79,7 @@ dynamic behavior and replace with static configuration.
         <username>root</username>
         <user_password>%%ROOTPASSWORD%%</user_password>
         <encrypted config:type="boolean">true</encrypted>
-	<!--INSERT:/tmp/rootkeys.xml-->
+	<xi:include href="file:///tmp/rootkeys.xml"/>
       </user>
     </users>
     <networking>
@@ -101,6 +99,7 @@ dynamic behavior and replace with static configuration.
          <pre-scripts config:type="list">
         <script>
           <filename>preinstall.sh</filename>
+          <interpreter>shell</interpreter>
           <source>
 <![CDATA[
 #!/bin/sh
@@ -116,6 +115,7 @@ curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.
          <chroot-scripts config:type="list">
         <script>
           <filename>chroot.sh</filename>
+          <interpreter>shell</interpreter>
           <source>
 <![CDATA[
 #!/bin/sh
@@ -135,12 +135,14 @@ cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
         <script>
           <chrooted config:type="boolean">true</chrooted>
           <filename>post.sh</filename>
+          <interpreter>shell</interpreter>
           <location>file:///etc/confluent/post.sh</location>
         </script>
       </chroot-scripts>
       <init-scripts config:type="list">
         <script>
           <filename>firstboot.sh</filename>
+          <interpreter>shell</interpreter>
           <location>file:///etc/confluent/firstboot.sh</location>
         </script>
      </init-scripts> 

confluent_osdeploy/suse15/profiles/hpc/initprofile.sh

@@ -2,22 +2,19 @@
 discnum=$(basename $1)
 if [ "$discnum" != 1 ]; then exit 0; fi
 if [ -e $2/boot/kernel ]; then exit 0; fi
-profile=$(basename $2)
-
-if [[ $profile =~ ^sle.* ]]; then
-    if ls $1/Product-* >& /dev/null; then
-        ln -s $1 $2/product
-    else
-        ln -s ${1%1}2 $2/product
-    fi
+if ls $1/Product-* >& /dev/null; then
+	ln -s $1 $2/product
+else
+	ln -s ${1%1}2 $2/product
 fi
 sed -i 's/sle 15/SUSE Linux Enterprise 15/; s/opensuse_leap/openSUSE Leap/' $2/profile.yaml
 ln -s $1/boot/x86_64/loader/linux $2/boot/kernel && \
 ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
 mkdir -p $2/boot/efi/boot && \
 ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
+profile=$(basename $2)
 if [[ $profile =~ ^sle.* ]]; then
 	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
 else
-	sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast
+	sed -i 's/%%IFSLE%%/<!--/;s/%%ENDIFSLE%%/-->/' $2/autoyast
 fi

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -10,9 +10,6 @@ confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 . /etc/confluent/functions
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
-	sleep 1
-done
 
 for i in  /etc/ssh/ssh_host*key.pub; do
     certname=${i/.pub/-cert.pub}

confluent_osdeploy/suse15/profiles/hpc/scripts/functions

@@ -1,7 +1,4 @@
 function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
     if [ -z "$confluent_mgr" ]; then
         confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi

confluent_osdeploy/suse15/profiles/hpc/scripts/pre.sh

@@ -38,10 +38,4 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
     ntpcfg=$(paste -sd '' /tmp/ntp.cfg)
 fi
 run_remote_python getinstalldisk
-sed -e s'!'%%INSTDISK%%'!'/dev/$(cat /tmp/installdisk)'!' -e s'!'%%NODENAME%%'!'$nodename'!' -e 's!<networking\(.*\)>!'"$ntpcfg"'<networking\1>!' -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml
-if grep append /tmp/bootloader.xml > /dev/null; then
-    sed -i 's@</general>@</general>'"$(tr -d '\n' < /tmp/bootloader.xml)"'@' /tmp/profile/modified.xml
-fi
-sed -i 's#root</username>#root</username>'"$(tr -d '\n' < /tmp/rootkeys.xml)"'#' /tmp/profile/modified.xml
-sed -i 's@/hwclock>@/hwclock>'"$(tr -d '\n' < /tmp/timezone)"'@' /tmp/profile/modified.xml
-sed -i 's@<media_url/>@'"$(tr -d '\n' < /tmp/pkgurl)"'@' /tmp/profile/modified.xml
+sed -e s!%%INSTDISK%%!/dev/$(cat /tmp/installdisk)! -e s!%%NODENAME%%!$nodename! -e "s!<networking>!$ntpcfg<networking>!" -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh

@@ -9,9 +9,6 @@ fi
 nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
-	sleep 1
-done
 hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 touch /etc/cloud/cloud-init.disabled
 source /etc/confluent/functions

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/functions

@@ -1,7 +1,4 @@
 function set_confluent_vars() {
-    if [ -z "$nodename" ]; then
-        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
-    fi
     if [ -z "$confluent_mgr" ]; then
         confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi

confluent_osdeploy/utils/Makefile

@@ -1,12 +1,8 @@
 CC := gcc
 CFLAGS := -Os
-TARGETS := copernicus autocons start_root confluent_imginfo
+TARGETS := copernicus autocons
 
-all: $(TARGETS) clortho urlmount
-
-urlmount: urlmount.c
-	$(CC) $(CFLAGS) -o $@ $^  -D_FILE_OFFSET_BITS=64  -lcurl -lm -lfuse -lpthread
-	strip -s $@
+all: $(TARGETS) clortho
 
 clortho: clortho.c
 	$(CC) $(CFLAGS) -o $@ $^ -lcrypt

confluent_osdeploy/utils/autocons.c

@@ -95,7 +95,7 @@ int main(int argc, char* argv[]) {
     timeout.tv_usec = 500000;
     FD_ZERO(&set);
     FD_SET(ttyf, &set);
-    if (write(ttyf, "\0337\033[999;999H\033[6n\0338", 18) < 0) {};
+    write(ttyf, "\0337\033[999;999H\033[6n\0338", 18);
     while (select(ttyf + 1, &set, NULL, NULL, &timeout) > 0) {
         if ((tmpi = read(ttyf, buff + bufflen, 127 - bufflen)) < 0) {
             if (errno == EAGAIN || errno == EWOULDBLOCK) {

confluent_osdeploy/utils/clortho.c

@@ -116,7 +116,7 @@ int main(int argc, char* argv[]) {
             currtype = buffer[0];
 	    if (currtype & 0b10000000) {
                 currlen = buffer[1] << 8;
-                while (read(sock, buffer, 1) < 0) {}; 
+                read(sock, buffer, 1);
                 currlen |= buffer[0];
             } else {
                 currlen = buffer[1];
@@ -131,7 +131,7 @@ int main(int argc, char* argv[]) {
                 buffer[currlen] = 0;
             }
             if (currtype == 2) {
-                dprintf(sock, "\x03%c", (int)currlen);
+                dprintf(sock, "\x03%c", currlen);
                 ret = write(sock, buffer, currlen);
                 slen = strlen(cryptedpass) & 0xff;
                 dprintf(sock, "\x04%c%s", slen, cryptedpass);

confluent_osdeploy/utils/confluent_imginfo.c

@@ -1,82 +0,0 @@
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-int read_part(FILE* img, long int imgsize) {
-    char mountpath[65537];
-    char devpath[65537];
-    char fstype[65537];
-    uint16_t shortlength;
-    uint32_t length;
-    uint64_t longlength;
-    fread(&shortlength, 2, 1, img);
-    shortlength = be16toh(shortlength);
-    fread(mountpath, 1, shortlength, img);
-    mountpath[shortlength] = 0;
-    fread(&length, 4, 1, img);
-    length = be32toh(length);
-    fseek(img, length, SEEK_CUR); // skip json section that we don't support
-    fread(&longlength, 8, 1, img); // minimum size in bytes
-    longlength = be64toh(longlength);
-    printf("%ld\t", longlength);
-    fread(&longlength, 8, 1, img); // default size in bytes
-    longlength = be64toh(longlength);
-    printf("%ld\t", longlength);
-    fread(&shortlength, 2, 1, img); // length of filesystem type
-    shortlength = be16toh(shortlength);
-    fread(fstype, 1, shortlength, img);
-    fstype[shortlength] = 0;
-    fread(&shortlength, 2, 1, img); // length of DEVICE
-    shortlength = be16toh(shortlength);
-    fread(devpath, 1, shortlength, img);
-    devpath[shortlength] = 0;
-    fread(&shortlength, 2, 1, img);
-    shortlength = be16toh(shortlength);
-    fseek(img, shortlength, SEEK_CUR); // Skip the padding
-    fread(&longlength, 8, 1, img);
-    longlength = be64toh(longlength);
-    printf("%ld\t", ftell(img) / 512);
-    printf("%ld\t", longlength / 512);
-    printf("%s\t%s\t%s\n", fstype, devpath, mountpath);
-    fseek(img, longlength, SEEK_CUR);
-    return (ftell(img) < imgsize);
-}
-
-int main(int argc, char* argv[]) {
-    FILE* img;
-    long int imgsize;
-    char buffer[20];
-
-    img = fopen(argv[1], "rb");
-    fseek(img, 0, SEEK_END);
-    imgsize = ftell(img);
-    fseek(img, 0, SEEK_SET);
-    if (fread(buffer, 1, 16, img) < 16) {
-        fprintf(stderr, "Unable to read header of image\n");
-        exit(1);
-    }
-    if ((memcmp(buffer, "hsqs", 4) == 0) || (memcmp(buffer, "sqsh", 4) == 0)) {
-        printf("Format: squashfs\n");
-        exit(0);
-    }
-    if (memcmp(buffer, "\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1", 16) == 0) {
-        printf("Format: confluent_multisquash\nminsize\tdefsize\toffset\tsize\tfstype\torigdev\tmount\n");
-        fread(buffer, 1, 1, img);
-        fseek(img, buffer[0], SEEK_CUR);
-        while (read_part(img, imgsize));
-        exit(0);
-    }
-    if (memcmp(buffer, "\xaa\xd5\x0f\x7e\x5d\xfb\x4b\x7c\xa1\x2a\xf4\x0b\x6d\x94\xf7\xfc", 16) == 0) {
-        fread(buffer, 1, 1, img);
-        fseek(img, buffer[0], SEEK_CUR);
-        fread(buffer, 1, 1, img);
-        if (buffer[0] == 0) {
-            printf("Format: confluent_crypted\n");
-            exit(0);
-        }
-    }
-    fprintf(stderr, "Unrecognized image format\n");
-    exit(1);
-
-}

confluent_osdeploy/utils/start_root.c

@@ -1,17 +0,0 @@
-#include <unistd.h>
-#include <stdio.h>
-#include <sys/mount.h>
-#define __USE_GNU
-#include <sched.h>
-int main(int argc, char* argv[]) {
-    unshare(CLONE_NEWNS);
-    mount("/dev", "/sysroot/dev", NULL, MS_MOVE, NULL);
-    mount("/proc", "/sysroot/proc", NULL, MS_MOVE, NULL);
-    mount("/sys", "/sysroot/sys", NULL, MS_MOVE, NULL);
-    mount("/run", "/sysroot/run", NULL, MS_MOVE, NULL);
-    if (chdir("/sysroot") < 0) { fprintf(stderr, "Unable to chdir!\n"); }
-    mount("/sysroot", "/", NULL, MS_MOVE, NULL);
-    if (chroot(".") < 0) { fprintf(stderr, "Failed to chroot!\n"); }
-    if (chdir("/") < 0) { fprintf(stderr, "Unable to chdir after chroot!\n"); }
-    execl("/sbin/init", "/sbin/init", NULL);
-}

confluent_osdeploy/utils/urlmount.c

@@ -1,283 +0,0 @@
-/*
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2021 Lenovo
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-#define FUSE_USE_VERSION 26
-#include <fuse.h>
-#include <curl/curl.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <pthread.h>
-#include <unistd.h>
-
-CURL *curl;
-
-char curlerror[CURL_ERROR_SIZE];
-curl_off_t filesize;
-
-typedef struct downloadbuffer {
-    char *response;
-    size_t completed;
-    size_t total;
-} downloadbuffer;
-
-#define MAX_FILE_LEN 1024
-#define MAX_URL_PATHS 512
-static char filename[MAX_FILE_LEN];
-static int urlidx, newidx;
-static char* urls[MAX_URL_PATHS];
-
-
-void *http_rechecker(void *argp) {
-    CURL *checkurl;
-    int tmpidx, tmpval;
-    tmpidx = open("/dev/urandom", O_RDONLY);
-    if (tmpidx <= 0 || read(tmpidx, (char*)&tmpval, 4) < 0)
-        tmpval = time(NULL);
-    if (tmpidx > 0)
-        close(tmpidx);
-    srand(tmpval);
-    checkurl = curl_easy_init();
-    curl_easy_setopt(checkurl, CURLOPT_ERRORBUFFER, curlerror);
-    //We want to consider error conditions fatal, rather than
-    //passing error text as data
-    curl_easy_setopt(checkurl, CURLOPT_FAILONERROR, 1L);
-    curl_easy_setopt(checkurl, CURLOPT_TIMEOUT, 10L);
-    curl_easy_setopt(checkurl, CURLOPT_NOBODY, 1);
-    while (1) {
-        sleep(25 + rand() % 10);  // Spread out retries across systems
-        tmpidx = 0;
-        while (tmpidx < urlidx && tmpidx < newidx && urls[tmpidx] != NULL) {
-            curl_easy_setopt(checkurl, CURLOPT_URL, urls[tmpidx]);
-            if (curl_easy_perform(checkurl) == CURLE_OK)
-                newidx = tmpidx;
-            else
-                tmpidx++;
-        }
-    }
-}
-
-static int http_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
-                        off_t offset, struct fuse_file_info *fi)
-{
-    if (strcmp(path, "/") != 0)  // We don't support subdirs
-        return -ENOENT;
-    filler(buf, ".", NULL, 0);
-    filler(buf, "..", NULL, 0);
-    filler(buf, filename + 1, NULL, 0);
-    return 0;
-}
-
-size_t fill_buffer(char *data, size_t size, size_t nmemb, downloadbuffer *userdata) {
-    size_t amount;
-    amount = size * nmemb;
-    if (userdata->total < amount + userdata->completed) return 0;
-    memcpy(&(userdata->response[userdata->completed]), data, amount);
-    userdata->completed += amount;
-    return amount;
-}
-
-static int http_read(const char *path, char *buf, size_t size, off_t offset,
-                     struct fuse_file_info *fi) {
-    char headbuffer[512];
-    double dldbl = 0.0;
-    int startidx;
-    int reconnecting = 0;
-    FILE* fd;
-    startidx = urlidx;
-    memset(buf, 0, size);
-    curl_off_t downloaded;
-    //Would be needed for multithread, however preferring to conserve
-    //filehandles rather than go multithread
-    // Some comparisons showed that the threaded performance boost doesn't
-    // do even offset the overhead of the new curl handles, so better
-    // to use single threaded curl overall for now
-    //CURL *tmpcurl = curl_easy_duphandle(curl);
-    downloadbuffer dlbuf;
-    dlbuf.response = buf;
-    dlbuf.completed = 0;
-    dlbuf.total = size;
-    fd = NULL;
-
-    if (strcmp(path, filename) != 0) return -ENOENT;
-    memset(headbuffer, 0, 512);
-    if (offset >= filesize) return 0;
-    if (offset + size - 1 >= filesize) size = filesize - offset - 1;
-    snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
-    curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer);
-    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dlbuf);
-    if (newidx < MAX_URL_PATHS) {
-        reconnecting = 1;
-        urlidx = newidx;
-        newidx = MAX_URL_PATHS;
-        fd = fopen("/dev/kmsg", "w+");
-        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
-        fclose(fd);
-        curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    }
-    while (curl_easy_perform(curl) != CURLE_OK) {
-        reconnecting = 1;
-        fd = fopen("/dev/kmsg", "w+");
-        dlbuf.completed = 0;
-        fprintf(fd, "<4>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror);
-        fclose(fd);
-        urlidx++;
-        if (urls[urlidx] == NULL)
-            urlidx = 0;
-        if (urlidx == startidx) {
-            fd = fopen("/dev/kmsg", "w+");
-            fprintf(fd, "<1>urlmount: All connections to source are down\n");
-            fclose(fd);
-            sleep(10);
-        }
-        fd = fopen("/dev/kmsg", "w+");
-        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
-        fclose(fd);
-        curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    }
-    if (reconnecting) {
-        fd = fopen("/dev/kmsg", "w+");
-        fprintf(fd, "<5>urlmount: Successfully connected to %s\n", urls[urlidx]);
-        fclose(fd);
-    }
-    curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dldbl);
-    downloaded = round(dldbl);
-    //Would be needed for multithread
-    //curl_easy_cleanup(tmpcurl);
-    return downloaded;
-}
-
-static int http_open(const char *path, struct fuse_file_info *fi) {
-    if (strcmp(path, filename) != 0)
-        return -ENOENT;
-
-    if ((fi->flags & 3) != O_RDONLY)
-        return -EACCES;
-
-    return 0;
-}
-
-static void* http_init(struct fuse_conn_info *conn) {
-    // Because we fork, we need to redo curl
-    // or else suffer the wrath of NSS TLS
-    pthread_t tid;
-    curl_global_init(CURL_GLOBAL_DEFAULT);
-    pthread_create(&tid, NULL, http_rechecker, NULL);
-    curl = curl_easy_init();
-    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
-    //We want to consider error conditions fatal, rather than
-    //passing error text as data
-    curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
-    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
-    curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer);
-    return NULL;
-}
-
-static int http_getattr(const char *path, struct stat *st) {
-    memset(st, 0, sizeof(struct stat));
-
-    if (strcmp(path, "/") == 0) {
-        st->st_mode = S_IFDIR | 0555;
-        st->st_nlink = 2;
-    } else if (strcmp(path, filename) == 0) {
-        st->st_mode = S_IFREG | 0444;
-        st->st_nlink = 1;
-        st->st_size = filesize; // TODO: fix with curl HEAD
-    } else
-        return -ENOENT;
-    return 0;
-}
-
-
-static const struct fuse_operations http_ops = {
-    .getattr = http_getattr,
-    .readdir = http_readdir,
-    .read = http_read,
-    .open = http_open,
-    .init = http_init,
-};
-
-int main(int argc, char* argv[]) {
-    char *tmp;
-    double fsize;
-    unsigned int i;
-    int j;
-    j = 0;
-    memset(urls, 0, 32*sizeof(char*));
-    urlidx = 0;
-    newidx = MAX_URL_PATHS;
-    curl_global_init(CURL_GLOBAL_DEFAULT);
-    curl = curl_easy_init();
-    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
-    //We want to consider error conditions fatal, rather than
-    //passing error text as data
-    curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
-    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L);
-    memset(filename, 0, MAX_FILE_LEN);
-    for (i=0; i < argc; i++) {
-        if (strstr(argv[i], ":") > 0) {
-            if (j < MAX_URL_PATHS) {
-                urls[j] = argv[i];
-                tmp = strrchr(urls[j++], '/');
-                strncpy(filename, tmp, MAX_FILE_LEN);
-            }
-            //Request single threaded mode, as curl would need more
-            // filehandles for multithread
-            argv[i] = "-s";
-        }
-    }
-    if (filename[0] == 0) {
-        fprintf(stderr, "No URL given in arguments\n");
-        exit(1);
-    }
-    for (i=0; urls[i] != NULL; i++) {
-        printf("Registering mount path: %s\n", urls[i]);
-    }
-    j = urlidx;
-    printf("Connecting to %s\n", urls[urlidx]);
-    curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
-    while (curl_easy_perform(curl) != CURLE_OK) {
-        fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror);
-        if (urls[urlidx] == NULL)
-            urlidx = 0;
-        if (urlidx == j) {
-            fprintf(stderr, "urlmount: Unable to reach any target url, aborting\n");
-            exit(1);
-        }
-        printf("Connecting to %s\n", urls[urlidx]);
-        curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    }
-    printf("Successfully connected to %s\n", urls[urlidx]);
-    curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize);
-    filesize = round(fsize);
-    curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
-    if (filesize < 1) {
-        fprintf(stderr, "Unable to reach designated URL\n");
-        exit(1);
-    }
-    if (!curl) {
-        fprintf(stderr, "Unable to initialize CURL!\n");
-        exit(1);
-    }
-    curl_easy_cleanup(curl);
-    curl_global_cleanup();
-    fuse_main(argc, argv, &http_ops, NULL);
-}

confluent_server/builddeb

@@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyte, python3-pyghmi, python3-paramiko/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides

confluent_server/confluent/auth.py

@@ -26,7 +26,6 @@ import Cryptodome.Protocol.KDF as KDF
 from fnmatch import fnmatch
 import hashlib
 import hmac
-import msgpack
 import multiprocessing
 import os
 import pwd
@@ -101,10 +100,6 @@ _deniedbyrole = {
 }
 
 
-class PromptsNeeded(Exception):
-    def __init__(self, prompts):
-        self.prompts = prompts
-
 def _get_usertenant(name, tenant=False):
     """_get_usertenant
 
@@ -226,14 +221,11 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
     if ucfg is None:
         eventlet.sleep(0.05)
         return None
-    bpassphrase = None
-    if isinstance(passphrase, dict) and len(passphrase) == 1:
-        passphrase = list(passphrase.values())[0]
     if isinstance(passphrase, bytes):
         bpassphrase = passphrase
-    elif not isinstance(passphrase, dict):
+    else:
         bpassphrase = passphrase.encode('utf8')
-    if (user, tenant) in _passcache and bpassphrase:
+    if (user, tenant) in _passcache:
         if hashlib.sha256(bpassphrase).digest() == _passcache[(user, tenant)]:
             return authorize(user, element, tenant, operation=operation)
         else:
@@ -241,7 +233,7 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
             # while someone is legitimately logged in
             # invalidate cache and force the slower check
             del _passcache[(user, tenant)]
-    if 'cryptpass' in ucfg and bpassphrase:
+    if 'cryptpass' in ucfg:
         _passchecking[(user, tenant)] = True
         # TODO(jbjohnso): WORKERPOOL
         # PBKDF2 is, by design, cpu intensive
@@ -283,41 +275,23 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
             # to let a non-0 user check anothers password.
             # We will fork and the child will assume elevated privilege to
             # get unix_chkpwd helper to enable checking /etc/shadow
-            getprompt, sendprompt = os.pipe()
-            getprompt, sendprompt = os.fdopen(getprompt, 'rb', 0), os.fdopen(sendprompt, 'wb', 0)
             pid = os.fork()
             if not pid:
                 usergood = False
                 try:
-                    getprompt.close()
                     # we change to the uid we are trying to authenticate as, because
                     # pam_unix uses unix_chkpwd which reque
                     os.setuid(pwe.pw_uid)
-                    pa = pam.pam()
-                    usergood = pa.authenticate(user, passphrase, service=_pamservice)
-                    if (not usergood and len(pa.prompts) > 1 and
-                            (not isinstance(passphrase, dict) or
-                            (set(passphrase) - pa.prompts))):
-                        sendprompt.write(msgpack.packb(list(pa.prompts)))
-                        sendprompt.close()
-                        os._exit(2)
+                    usergood = pam.authenticate(user, passphrase, service=_pamservice)
                 finally:
                     os._exit(0 if usergood else 1)
-            sendprompt.close()
-            usergood = os.waitpid(pid, 0)[1]
-            if (usergood >> 8) == 2:
-                prompts = getprompt.read()
-                if (prompts):
-                    raise PromptsNeeded(msgpack.unpackb(prompts))
-            usergood = usergood == 0
-            getprompt.close()
+            usergood = os.waitpid(pid, 0)[1] == 0
         else:
             # We are running as root, we don't need to fork in order to authenticate the
             # user
             usergood = pam.authenticate(user, passphrase, service=_pamservice)
         if usergood:
-            if bpassphrase:
-                _passcache[(user, tenant)] = hashlib.sha256(bpassphrase).digest()
+            _passcache[(user, tenant)] = hashlib.sha256(bpassphrase).digest()
             return authorize(user, element, tenant, operation, skipuserobj=False)
     eventlet.sleep(0.05)  # stall even on test for existence of a username
     return None

confluent_server/confluent/certutil.py

@@ -66,29 +66,33 @@ def get_certificate_paths():
 
     return keypath, certpath
 
-def assure_tls_ca():
-    keyout, certout = ('/etc/confluent/tls/cakey.pem', '/etc/confluent/tls/cacert.pem')
-    if os.path.exists(certout):
-        return
-    try:
-        os.makedirs('/etc/confluent/tls')
-    except OSError as e:
-        if e.errno != 17:
-            raise
-    sslcfg = get_openssl_conf_location()
-    tmpconfig = tempfile.mktemp()
-    shutil.copy2(sslcfg, tmpconfig)
+def create_certificate(keyout=None, certout=None):
+    if not keyout:
+        keyout, certout = get_certificate_paths()
+    if not keyout:
+        raise Exception('Unable to locate TLS certificate path automatically')
+    shortname = socket.gethostname().split('.')[0]
+    longname = shortname # socket.getfqdn()
     subprocess.check_call(
         ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
          keyout])
+    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
+    # It is incorrect to put IP addresses as DNS type.  However
+    # there exists non-compliant clients that fail with them as IP
+    san.extend(['DNS:{0}'.format(x) for x in get_ip_addresses()])
+    san.append('DNS:{0}'.format(shortname))
+    #san.append('DNS:{0}'.format(longname))
+    san = ','.join(san)
+    sslcfg = get_openssl_conf_location()
+    tmpconfig = tempfile.mktemp()
+    shutil.copy2(sslcfg, tmpconfig)
     try:
         with open(tmpconfig, 'a') as cfgfile:
-            cfgfile.write('\n[CACert]\nbasicConstraints = CA:true\n')
+            cfgfile.write('\n[SAN]i\nbasicConstraints = CA:true\nsubjectAltName={0}'.format(san))
         subprocess.check_call([
             'openssl', 'req', '-new', '-x509', '-key', keyout, '-days',
-            '27300', '-out', certout, '-subj',
-            '/CN=Confluent TLS Certificate authority ({0})'.format(socket.gethostname()),
-            '-extensions', 'CACert', '-config', tmpconfig
+            '7300', '-out', certout, '-subj', '/CN={0}'.format(longname),
+            '-extensions', 'SAN', '-config', tmpconfig
         ])
     finally:
         os.remove(tmpconfig)
@@ -100,9 +104,9 @@ def assure_tls_ca():
     except OSError as e:
         if e.errno != 17:
             raise
-    shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
+    shutil.copy2(certout, fname)
     hv = subprocess.check_output(
-        ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
+        ['openssl', 'x509', '-in', certout, '-hash', '-noout'])
     if not isinstance(hv, str):
         hv = hv.decode('utf8')
     hv = hv.strip()
@@ -119,52 +123,6 @@ def assure_tls_ca():
                 pass
     os.symlink(certname, hashname)
 
-def create_certificate(keyout=None, certout=None):
-    if not keyout:
-        keyout, certout = get_certificate_paths()
-    if not keyout:
-        raise Exception('Unable to locate TLS certificate path automatically')
-    assure_tls_ca()
-    shortname = socket.gethostname().split('.')[0]
-    longname = shortname # socket.getfqdn()
-    subprocess.check_call(
-        ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
-         keyout])
-    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
-    # It is incorrect to put IP addresses as DNS type.  However
-    # there exists non-compliant clients that fail with them as IP
-    san.extend(['DNS:{0}'.format(x) for x in get_ip_addresses()])
-    san.append('DNS:{0}'.format(shortname))
-    #san.append('DNS:{0}'.format(longname))
-    san = ','.join(san)
-    sslcfg = get_openssl_conf_location()
-    tmpconfig = tempfile.mktemp()
-    extconfig = tempfile.mktemp()
-    csrout = tempfile.mktemp()
-    shutil.copy2(sslcfg, tmpconfig)
-    try:
-        with open(tmpconfig, 'a') as cfgfile:
-            cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san))
-        with open(extconfig, 'a') as cfgfile:
-            cfgfile.write('\nbasicConstraints=CA:false\nsubjectAltName={0}'.format(san))
-        subprocess.check_call([
-            'openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj',
-            '/CN={0}'.format(longname),
-            '-extensions', 'SAN', '-config', tmpconfig
-        ])
-        subprocess.check_call([
-            'openssl', 'x509', '-req', '-in', csrout,
-            '-CA', '/etc/confluent/tls/cacert.pem',
-            '-CAkey', '/etc/confluent/tls/cakey.pem',
-            '-set_serial', '0x123', '-out', certout, '-days', '27300',
-            '-extfile', extconfig
-        ])
-    finally:
-        os.remove(tmpconfig)
-        os.remove(csrout)
-        os.remove(extconfig)
-
-
 if __name__ == '__main__':
     outdir = os.getcwd()
     keyout = os.path.join(outdir, 'key.pem')

confluent_server/confluent/config/attributes.py

@@ -461,26 +461,6 @@ node = {
                         'case for other net attributes, net.eth0.ipv4_gateway '
                         'and similar is accepted.'
     },
-    'net.ipv6_address': {
-        'description': 'When configuring static, use this address.  If '
-                       'unspecified, it will check if the node name resolves '
-                       'to an IP address.  Additionally, the subnet prefix '
-                       'may be specified with a suffix, e.g. "/64".  If not '
-                       'specified, it will attempt to autodetect based on '
-                       'current network configuration.'
-    },
-    'net.ipv6_method': {
-        'description': 'Whether to use static or dhcp when configuring this '
-                       'interface for IPv6. "firmwaredhcp" means to defer to '
-                       'external DHCP server during firmware execution, but '
-                       'use static for OS',
-        'validvalues': ('dhcp', 'static', 'firmwaredhcp', 'none')
-    },
-    'net.ipv6_gateway': {
-        'description':  'The IPv6 gateway to use if applicable.  As is the '
-                        'case for other net attributes, net.eth0.ipv6_gateway '
-                        'and similar is accepted.'
-    },
     'net.hwaddr': {
         'description': 'The hardware address, aka MAC address of the interface indicated, generally populated by the '
                        'PXE discovery mechanism'

confluent_server/confluent/consoleserver.py

@@ -143,7 +143,6 @@ class ConsoleHandler(object):
         #self.termstream.attach(self.buffer)
         self.livesessions = set([])
         self.utf8decoder = codecs.getincrementaldecoder('utf-8')()
-        self.pendingbytes = None
         if self._logtobuffer:
             self.logger = log.Logger(node, console=True,
                                      tenant=configmanager.tenant)
@@ -187,15 +186,8 @@ class ConsoleHandler(object):
     def feedbuffer(self, data):
         if not isinstance(data, bytes):
             data = data.encode('utf-8')
-        if self.pendingbytes is not None:
-            self.pendingbytes += data
-        self.pendingbytes = b''
         try:
             send_output(self.node, data)
-            data = self.pendingbytes
-            self.pendingbytes = None
-            if data:
-                send_output(self.node, data)
         except Exception:
             _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                           event=log.Events.stacktrace)
@@ -658,7 +650,6 @@ class ProxyConsole(object):
         while data:
             self.data_handler(data)
             data = tlvdata.recv(self.remote)
-        self.remote.close()
 
     def get_buffer_age(self):
         # the server sends a buffer age if appropriate, no need to handle

confluent_server/confluent/core.py

@@ -77,15 +77,13 @@ try:
 except NameError:
     unicode = str
 
-def seek_element(currplace, currkey, depth):
+def seek_element(currplace, currkey):
     try:
         return currplace[currkey]
     except TypeError:
         if isinstance(currplace, PluginCollection):
             # we hit a plugin curated collection, all children
             # are up to the plugin to comprehend
-            if currplace.maxdepth and depth > currplace.maxdepth:
-                raise
             return currplace
         raise
 
@@ -93,9 +91,8 @@ def seek_element(currplace, currkey, depth):
 def nested_lookup(nestdict, key):
     try:
         currloc = nestdict
-        for i in range(len(key)):
-            currk = key[i]
-            currloc = seek_element(currloc, currk, len(key) - i)
+        for currk in key:
+            currloc = seek_element(currloc, currk)
         return currloc
     except TypeError:
         raise exc.NotFoundException("Invalid element requested")
@@ -146,9 +143,8 @@ class PluginRoute(object):
 
 
 class PluginCollection(object):
-    def __init__(self, routedict, maxdepth=1):
+    def __init__(self, routedict):
         self.routeinfo = routedict
-        self.maxdepth = maxdepth
 
 
 def handle_deployment(configmanager, inputdata, pathcomponents,
@@ -266,10 +262,6 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
-                'location': PluginRoute({
-                    'pluginattrs': ['hardwaremanagement.method'],
-                    'default': 'ipmi',
-                }),
                 'ntp': {
                     'enabled': PluginRoute({
                         'pluginattrs': ['hardwaremanagement.method'],
@@ -357,10 +349,6 @@ def _init_core():
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-            'graphical': PluginRoute({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
         },
         'description': PluginRoute({
             'pluginattrs': ['hardwaremanagement.method'],
@@ -825,7 +813,6 @@ def handle_dispatch(connection, cert, dispatch, peername):
             _forward_rsp(connection, res)
     keepalive.kill()
     connection.sendall('\x00\x00\x00\x00\x00\x00\x00\x00')
-    connection.close()
 
 
 def _forward_rsp(connection, res):
@@ -1054,12 +1041,12 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
         remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                  keyfile='/etc/confluent/privkey.pem',
                                  certfile='/etc/confluent/srvcert.pem')
-    except Exception as e:
+    except Exception:
         for node in nodes:
             if a:
                 yield msg.ConfluentResourceUnavailable(
-                    node, 'Collective member {0} is unreachable ({1})'.format(
-                        a['name'], str(e)))
+                    node, 'Collective member {0} is unreachable'.format(
+                        a['name']))
             else:
                 yield msg.ConfluentResourceUnavailable(
                     node,

confluent_server/confluent/discovery/core.py

@@ -711,25 +711,13 @@ def detected(info):
     nodename, info['maccount'] = get_nodename(cfg, handler, info)
     if nodename and handler and handler.https_supported:
         dp = cfg.get_node_attributes([nodename],
-                                     ('pubkeys.tls_hardwaremanager', 'id.uuid', 'discovery.policy'))
-        dp = dp.get(nodename, {})
-        lastfp = dp.get('pubkeys.tls_hardwaremanager',
+                                     ('pubkeys.tls_hardwaremanager',))
+        lastfp = dp.get(nodename, {}).get('pubkeys.tls_hardwaremanager',
                                           {}).get('value', None)
         if util.cert_matches(lastfp, handler.https_cert):
             info['nodename'] = nodename
             known_nodes[nodename][info['hwaddr']] = info
             info['discostatus'] = 'discovered'
-            uuid = info.get('uuid', None)
-            if uuid:
-                storeuuid = dp.get('id.uuid', {}).get('value', None)
-                if not storeuuid:
-                    discop = dp.get('discovery.policy', {}).get('value', '')
-                    if discop:
-                        policies = set(discop.split(','))
-                    else:
-                        policies = set([])
-                    if policies & {'open', 'permissive'}:
-                        cfg.set_node_attributes({nodename: {'id.uuid': info['uuid']}})
             return  # already known, no need for more
     #TODO(jjohnson2): We might have to get UUID for certain searches...
     #for now defer probe until inside eval_node.  We might not have
@@ -862,9 +850,6 @@ def get_nodename(cfg, handler, info):
             if nodename is None:
                 _map_unique_ids()
                 nodename = nodes_by_uuid.get(curruuid, None)
-    if not nodename and info['handler'] == pxeh:
-        enrich_pxe_info(info)
-        nodename = info.get('nodename', None)
     if not nodename:
         # Ok, see if it is something with a chassis-uuid and discover by
         # chassis
@@ -927,13 +912,6 @@ def get_nodename_from_chained_smms(cfg, handler, info):
                 nodename = newnodename
     return nodename
 
-def get_node_guess_by_uuid(uuid):
-    for mac in known_uuids.get(uuid, {}):
-        nodename = known_uuids[uuid][mac].get('nodename', None)
-        if nodename:
-            return nodename
-    return None
-
 def get_node_by_uuid_or_mac(uuidormac):
     node = pxe.macmap.get(uuidormac, None)
     if node is not None:
@@ -1112,7 +1090,7 @@ def discover_node(cfg, handler, info, nodename, manual):
         del unknown_info[info['hwaddr']]
     info['discostatus'] = 'identified'
     dp = cfg.get_node_attributes(
-        [nodename], ('discovery.policy', 'id.uuid',
+        [nodename], ('discovery.policy',
                      'pubkeys.tls_hardwaremanager'))
     policy = dp.get(nodename, {}).get('discovery.policy', {}).get(
         'value', None)
@@ -1185,19 +1163,10 @@ def discover_node(cfg, handler, info, nodename, manual):
         except KeyError:
             pass
         return True
-    if info['handler'] == pxeh:
-        olduuid = dp.get(nodename, {}).get('id.uuid', {}).get(
-            'value', None)
-        if olduuid.lower() != info['uuid']:
-            log.log({'info': 'Detected {0}, but discovery.policy is not set to a '
-                            'value allowing discovery (open, permissive, or pxe)'.format(
-                                nodename)})
-            info['discofailure'] = 'policy'
-    else:
-        log.log({'info': 'Detected {0}, but discovery.policy is not set to a '
-                         'value allowing discovery (open or permissive)'.format(
-                            nodename)})
-        info['discofailure'] = 'policy'
+    log.log({'info': 'Detected {0}, but discovery.policy is not set to a '
+                     'value allowing discovery (open or permissive)'.format(
+                        nodename)})
+    info['discofailure'] = 'policy'
     return False
 
 
@@ -1348,7 +1317,7 @@ def stop_autosense():
 
 def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
-    autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
+    autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe))
 
 
 nodes_by_fprint = {}

confluent_server/confluent/discovery/handlers/xcc.py

@@ -340,10 +340,6 @@ class NodeHandler(immhandler.NodeHandler):
                 rsp, status = wc.grab_json_response_with_status(
                     '/api/function',
                     {'USER_UserModify': '{0},{1},,1,Administrator,0,0,0,0,,8,'.format(uid, username)})
-            elif status == 200 and rsp.get('return', 0) == 13:
-                rsp, status = wc.grab_json_response_with_status(
-                    '/api/function',
-                    {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)})
             self.tmppasswd = None
         self._currcreds = (username, passwd)
 

confluent_server/confluent/discovery/protocols/pxe.py

@@ -25,10 +25,8 @@
 import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.noderange as noderange
-import confluent.neighutil as neighutil
 import confluent.log as log
 import confluent.netutil as netutil
-import confluent.util as util
 import ctypes
 import ctypes.util
 import eventlet
@@ -36,20 +34,13 @@ import eventlet.green.socket as socket
 import eventlet.green.select as select
 import netifaces
 import struct
-import time
 import traceback
-import uuid
 
 libc = ctypes.CDLL(ctypes.util.find_library('c'))
 
 iphdr = b'\x45\x00\x00\x00\x00\x00\x00\x00\x40\x11\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff'
 constiphdrsum = b'\x85\x11'
 udphdr = b'\x00\x43\x00\x44\x00\x00\x00\x00'
-ignoremacs = {}
-ignoredisco = {}
-
-mcastv6addr = 'ff02::1:2'
-
 
 def _ipsum(data):
     currsum = 0
@@ -195,26 +186,6 @@ def _decode_ocp_vivso(rq, idx, size):
         idx += rq[idx + 1] + 2
     return '', None, vivso
 
-def v6opts_to_dict(rq):
-    optidx = 0
-    reqdict = {}
-    disco = {'uuid':None, 'arch': None, 'vivso': None}
-    try:
-        while optidx < len(rq):
-            optnum, optlen = struct.unpack('!HH', rq[optidx:optidx+4])
-            reqdict[optnum] = rq[optidx + 4:optidx + 4 + optlen]
-            optidx += optlen + 4
-    except IndexError:
-        pass
-    if 1 in reqdict:
-        duid = reqdict[1]
-        if struct.unpack('!H', duid[:2])[0] == 4:
-            disco['uuid'] = decode_uuid(duid[2:])
-    if 61 in reqdict:
-        arch = bytes(rq[optidx+4:optidx+4+optlen])
-        disco['arch'] = pxearchs.get(bytes(reqdict[61]), None)
-    return reqdict, disco
-
 def opts_to_dict(rq, optidx, expectype=1):
     reqdict = {}
     disco = {'uuid':None, 'arch': None, 'vivso': None}
@@ -327,7 +298,7 @@ def start_proxydhcp():
     eventlet.spawn_n(proxydhcp)
 
 
-def snoop(handler, protocol=None, nodeguess=None):
+def snoop(handler, protocol=None):
     #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
     #prominent
     #TODO(jjohnson2): enable unicast replies. This would suggest either
@@ -343,130 +314,94 @@ def snoop(handler, protocol=None, nodeguess=None):
     net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
     net4.setsockopt(socket.IPPROTO_IP, IP_PKTINFO, 1)
     net4.bind(('', 67))
-    v6addr = socket.inet_pton(socket.AF_INET6, mcastv6addr)
-    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
-    net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    for ifidx in util.list_interface_indexes():
-        v6grp = v6addr + struct.pack('=I', ifidx)
-        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
-    net6.bind(('', 547))
-    clientaddr = sockaddr_in()
-    rawbuffer = bytearray(2048)
-    data = pkttype.from_buffer(rawbuffer)
-    msg = msghdr()
-    cmsgarr = bytearray(cmsgsize)
-    cmsg = cmsgtype.from_buffer(cmsgarr)
-    iov = iovec()
-    iov.iov_base = ctypes.addressof(data)
-    iov.iov_len = 2048
-    msg.msg_iov = ctypes.pointer(iov)
-    msg.msg_iovlen = 1
-    msg.msg_control = ctypes.addressof(cmsg)
-    msg.msg_controllen = ctypes.sizeof(cmsg)
-    msg.msg_name = ctypes.addressof(clientaddr)
-    msg.msg_namelen = ctypes.sizeof(clientaddr)
-    # We'll leave name and namelen blank for now
     while True:
         try:
             # Just need some delay, picked a prime number so that overlap with other
             # timers might be reduced, though it really is probably nothing
-            ready = select.select([net4, net6], [], [], None)
+            ready = select.select([net4], [], [], None)
             if not ready or not ready[0]:
                 continue
-            for netc in ready[0]:
-                idx = None
-                if netc == net4:
-                    i = recvmsg(netc.fileno(), ctypes.pointer(msg), 0)
-                    # if we have a small packet, just skip, it can't possible hold enough
-                    # data and avoids some downstream IndexErrors that would be messy
-                    # with try/except
-                    if i < 64:
-                        continue
-                    _, level, typ = struct.unpack('QII', cmsgarr[:16])
-                    if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
-                        idx, recv = struct.unpack('II', cmsgarr[16:24])
-                        recv = ipfromint(recv)
-                    rqv = memoryview(rawbuffer)
-                    if rawbuffer[0] == 1:  # Boot request
-                        process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv)
-                elif netc == net6:
-                    recv = 'ff02::1:2'
-                    pkt, addr = netc.recvfrom(2048)
-                    idx = addr[-1]
-                    i = len(pkt)
-                    if i < 64:
-                        continue
-                    rqv = memoryview(pkt)
-                    rq = bytearray(rqv[:2])
-                    if rq[0] in (1, 3): # dhcpv6 solicit
-                        process_dhcp6req(handler, rqv, addr, netc, cfg, nodeguess)
-
+            clientaddr = sockaddr_in()
+            rawbuffer = bytearray(2048)
+            data = pkttype.from_buffer(rawbuffer)
+            msg = msghdr()
+            cmsgarr = bytearray(cmsgsize)
+            cmsg = cmsgtype.from_buffer(cmsgarr)
+            iov = iovec()
+            iov.iov_base = ctypes.addressof(data)
+            iov.iov_len = 2048
+            msg.msg_iov = ctypes.pointer(iov)
+            msg.msg_iovlen = 1
+            msg.msg_control = ctypes.addressof(cmsg)
+            msg.msg_controllen = ctypes.sizeof(cmsg)
+            msg.msg_name = ctypes.addressof(clientaddr)
+            msg.msg_namelen = ctypes.sizeof(clientaddr)
+            # We'll leave name and namelen blank for now
+            i = recvmsg(net4.fileno(), ctypes.pointer(msg), 0)
+            # if we have a small packet, just skip, it can't possible hold enough
+            # data and avoids some downstream IndexErrors that would be messy
+            # with try/except
+            if i < 64:
+                continue
+            #peer = ipfromint(clientaddr.sin_addr.s_addr)
+            # We don't need peer yet, generally it's 0.0.0.0
+            _, level, typ = struct.unpack('QII', cmsgarr[:16])
+            if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
+                idx, recv, targ = struct.unpack('III', cmsgarr[16:28])
+                recv = ipfromint(recv)
+                targ = ipfromint(targ)
+            # peer is the source ip (in dhcpdiscover, 0.0.0.0)
+            # recv is the 'ip' that recevied the packet, regardless of target
+            # targ is the ip in the destination ip of the header.
+            # idx is the ip link number of the receiving nic
+            # For example, a DHCPDISCOVER will probably have:
+            # peer of 0.0.0.0
+            # targ of 255.255.255.255
+            # recv of <actual ip address that could reply>
+            # idx correlated to the nic
+            rqv = memoryview(rawbuffer)
+            rq = bytearray(rqv[:i])
+            if rq[0] == 1:  # Boot request
+                addrlen = rq[2]
+                if addrlen > 16 or addrlen == 0:
+                    continue
+                rawnetaddr = rq[28:28+addrlen]
+                netaddr = ':'.join(['{0:02x}'.format(x) for x in rawnetaddr])
+                optidx = 0
+                try:
+                    optidx = rq.index(b'\x63\x82\x53\x63') + 4
+                except ValueError:
+                    continue
+                txid = rq[4:8] # struct.unpack('!I', rq[4:8])[0]
+                rqinfo, disco = opts_to_dict(rq, optidx)
+                vivso = disco.get('vivso', None)
+                if vivso:
+                    # info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
+                    info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                            'architecture': vivso.get('arch', ''),
+                            'services': (vivso['service-type'],),
+                            'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid},
+                            'attributes': {'enclosure-machinetype-model': [vivso.get('machine', '')]}}
+                    handler(info)
+                    #consider_discover(info, rqinfo, net4, cfg, rqv)
+                    continue
+                # We will fill out service to have something to byte into,
+                # but the nature of the beast is that we do not have peers,
+                # so that will not be present for a pxe snoop
+                info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                        'architecture': disco['arch'],
+                        'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid},
+                        'services': ('pxe-client',)}
+                if disco['uuid']:  #TODO(jjohnson2): need to explictly check for
+                                # discover, so that the parser can go ahead and
+                                # parse the options including uuid to enable
+                                # ACK
+                    handler(info)
+                consider_discover(info, rqinfo, net4, cfg, rqv)
         except Exception as e:
             tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                             event=log.Events.stacktrace)
 
-def process_dhcp6req(handler, rqv, addr, net, cfg, nodeguess):
-    ip = addr[0]
-    req, disco = v6opts_to_dict(bytearray(rqv[4:]))
-    req['txid'] = rqv[1:4]
-    req['rqtype'] = bytearray(rqv[:1])[0]
-    if not disco.get('uuid', None) or not disco.get('arch', None):
-        return
-    mac = neighutil.get_hwaddr(ip.split('%', 1)[0])
-    if not mac:
-        net.sendto(b'\x00', addr)
-    tries = 5
-    while tries and not mac:
-        eventlet.sleep(0.01)
-        tries -= 1
-        mac = neighutil.get_hwaddr(ip.split('%', 1)[0])
-    info = {'hwaddr': mac, 'uuid': disco['uuid'],
-            'architecture': disco['arch'], 'services': ('pxe-client',)}
-    if ignoredisco.get(mac, 0) + 90 < time.time():
-        ignoredisco[mac] = time.time()
-        handler(info)
-    consider_discover(info, req, net, cfg, None, nodeguess, addr)
-
-def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv):
-    rq = bytearray(rqv)
-    addrlen = rq[2]
-    if addrlen > 16 or addrlen == 0:
-        return
-    rawnetaddr = rq[28:28+addrlen]
-    netaddr = ':'.join(['{0:02x}'.format(x) for x in rawnetaddr])
-    optidx = 0
-    try:
-        optidx = rq.index(b'\x63\x82\x53\x63') + 4
-    except ValueError:
-        return
-    txid = rq[4:8] # struct.unpack('!I', rq[4:8])[0]
-    rqinfo, disco = opts_to_dict(rq, optidx)
-    vivso = disco.get('vivso', None)
-    if vivso:
-                        # info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
-        info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
-                                'architecture': vivso.get('arch', ''),
-                                'services': (vivso['service-type'],),
-                                'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid},
-                                'attributes': {'enclosure-machinetype-model': [vivso.get('machine', '')]}}
-        if time.time() > ignoredisco.get(netaddr, 0) + 90:
-            ignoredisco[netaddr] = time.time()
-            handler(info)
-                        #consider_discover(info, rqinfo, net4, cfg, rqv)
-        return
-                    # We will fill out service to have something to byte into,
-                    # but the nature of the beast is that we do not have peers,
-                    # so that will not be present for a pxe snoop
-    info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
-                            'architecture': disco['arch'],
-                            'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid},
-                            'services': ('pxe-client',)}
-    if (disco['uuid']
-                            and time.time() > ignoredisco.get(netaddr, 0) + 90):
-        ignoredisco[netaddr] = time.time()
-        handler(info)
-    consider_discover(info, rqinfo, net4, cfg, rqv, nodeguess)
-
 
 
 def clear_nodes(nodes):
@@ -516,105 +451,13 @@ def get_deployment_profile(node, cfg, cfd=None):
 
 staticassigns = {}
 myipbypeer = {}
-def check_reply(node, info, packet, sock, cfg, reqview, addr):
+def check_reply(node, info, packet, sock, cfg, reqview):
     httpboot = info['architecture'] == 'uefi-httpboot'
+    replen = 275  # default is going to be 286
     cfd = cfg.get_node_attributes(node, ('deployment.*'))
     profile = get_deployment_profile(node, cfg, cfd)
     if not profile:
-        if time.time() > ignoremacs.get(info['hwaddr'], 0) + 90:
-            ignoremacs[info['hwaddr']] = time.time()
-            log.log({'info': 'Ignoring boot attempt by {0} no deployment profile specified (uuid {1}, hwaddr {2})'.format(
-                node, info['uuid'], info['hwaddr']
-            )})
         return
-    if addr:
-        if not httpboot:
-            log.log({'info': 'IPv6 PXE boot attempt by {0}, but IPv6 PXE is not supported, try IPv6 HTTP boot or IPv4 boot'.format(node)})
-            return
-        return reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock)
-    else:
-        return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile)
-
-def reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock):
-    myaddrs = netutil.get_my_addresses(addr[-1], socket.AF_INET6)
-    if not myaddrs:
-        log.log({'info': 'Unable to provide IPv6 boot services to {0}, no viable IPv6 configuration on interface index "{1}" to respond through.'.format(node, addr[-1])})
-        return
-    niccfg = netutil.get_nic_config(cfg, node, ifidx=addr[-1])
-    ipv6addr = niccfg.get('ipv6_address', None)
-    ipv6prefix = niccfg.get('ipv6_prefix', None)
-    ipv6method = niccfg.get('ipv6_method', 'static')
-    ipv6srvaddr = niccfg.get('deploy_server_v6', None)
-    if not ipv6srvaddr:
-        log.log({'info': 'Unable to determine an appropriate ipv6 server ip for {}'.format(node)})
-        return
-    insecuremode = cfd.get(node, {}).get('deployment.useinsecureprotocols',
-        {}).get('value', 'never')
-    if not insecuremode:
-        insecuremode = 'never'
-    proto = 'https' if insecuremode == 'never' else 'http'
-    bootfile = '{0}://[{1}]/confluent-public/os/{2}/boot.img'.format(
-        proto, ipv6srvaddr, profile
-    )
-    if not isinstance(bootfile, bytes):
-        bootfile = bootfile.encode('utf8')
-    ipass = []
-    if ipv6method not in ('dhcp', 'firmwaredhcp') and ipv6addr:
-        if not ipv6prefix:
-            log.log({'info': 'Unable to determine prefix to serve to address {} for node {}'.format(ipv6addr, node)})
-            return
-        ipass = bytearray(40)
-        ipass[:4] = packet[3][:4]  # pass iaid back
-        ipass[4:16] = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x18'
-        ipass[16:32] = socket.inet_pton(socket.AF_INET6, ipv6addr)
-        ipass[32:40] = b'\x00\x00\x00\x78\x00\x00\x01\x2c'
-    #1 msgtype
-    #3 txid
-    #22 - server ident
-    #len(packet[1]) + 4 - client ident
-    #len(ipass) + 4 or 0
-    #len(url) + 4
-    replylen = 50 + len(bootfile) + len(packet[1]) + 4
-    if len(ipass):
-        replylen += len(ipass)
-    reply = bytearray(replylen)
-    reply[0] = 2 if packet['rqtype'] == 1 else 7
-    reply[1:4] = packet['txid']
-    offset = 4
-    struct.pack_into('!HH', reply, offset, 1, len(packet[1]))
-    offset += 4
-    reply[offset:offset+len(packet[1])] = packet[1]
-    offset += len(packet[1])
-    struct.pack_into('!HHH', reply, offset, 2, 18, 4)
-    offset += 6
-    reply[offset:offset+16] = get_my_duid()
-    offset += 16
-    if ipass:
-        struct.pack_into('!HH', reply, offset, 3, len(ipass))
-        offset += 4
-        reply[offset:offset + len(ipass)] = ipass
-        offset += len(ipass)
-    struct.pack_into('!HH', reply, offset, 59, len(bootfile))
-    offset += 4
-    reply[offset:offset + len(bootfile)] = bootfile
-    offset += len(bootfile)
-    # Need the HTTPClient in the vendor class for reply
-    struct.pack_into('!HHIH', reply, offset, 16, 16, 0, 10)
-    offset += 10
-    reply[offset:offset + 10] = b'HTTPClient'
-    sock.sendto(reply, addr)
-
-
-_myuuid = None
-def get_my_duid():
-    global _myuuid
-    if not _myuuid:
-        _myuuid = uuid.uuid4().bytes
-    return _myuuid
-
-
-def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile):
-    replen = 275  # default is going to be 286
     myipn = info['netinfo']['recvip']
     myipn = socket.inet_aton(myipn)
 
@@ -788,40 +631,13 @@ def ack_request(pkt, rq, info):
     repview[26:28] = struct.pack('!H', datasum)
     send_raw_packet(repview, len(rply), rq, info)
 
-def consider_discover(info, packet, sock, cfg, reqview, nodeguess, addr=None):
+def consider_discover(info, packet, sock, cfg, reqview):
     if info.get('hwaddr', None) in macmap and info.get('uuid', None):
-        check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview, addr)
+        check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview)
     elif info.get('uuid', None) in uuidmap:
-        check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview, addr)
+        check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview)
     elif packet.get(53, None) == b'\x03':
         ack_request(packet, reqview, info)
-    elif info.get('uuid', None) and info.get('hwaddr', None):
-        if time.time() > ignoremacs.get(info['hwaddr'], 0) + 90:
-            ignoremacs[info['hwaddr']] = time.time()
-            maybenode = None
-            if nodeguess:
-                maybenode = nodeguess(info['uuid'])
-            if maybenode:
-                # originally was going to just offer up the node (the likely
-                # scenario is that it was manually added, autodiscovery picked
-                # up the TLS match, and correlated)
-                # However, since this is technically unverified data, we shouldn't
-                # act upon it until confirmed by process or user
-                # So instead, offer a hint about what is probably the case, but
-                # hasn't yet been approved by anything
-                log.log(
-                        {'info': 'Boot attempt from uuid {0} or hardware '
-                                 'address {1}, which is not confirmed to be a '
-                                 'node, but seems to be {2}. To confirm node '
-                                 'identity, \'nodediscover reassign -n {2}\' or '
-                                 '\'nodeattrib {2} id.uuid={0}\''.format(
-                            info['uuid'], info['hwaddr'], maybenode
-                )})
-            else:
-                log.log(
-                        {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
-                            info['uuid'], info['hwaddr']
-                )})
 
 
 if __name__ == '__main__':

confluent_server/confluent/discovery/protocols/slp.py

@@ -106,9 +106,12 @@ def _parse_slp_packet(packet, peer, rsps, xidmap):
     addr = peer[0]
     if '%' in addr:
         addr = addr[:addr.index('%')]
-    mac = neighutil.get_hwaddr(addr)
-    if mac:
-        identifier = mac
+    mac = None
+    if addr not in neighutil.neightable:
+        neighutil.update_neigh()
+    if addr in neighutil.neightable:
+        identifier = neighutil.neightable[addr]
+        mac = identifier
     else:
         identifier = addr
     if (identifier, parsed['xid']) in rsps:
@@ -251,6 +254,7 @@ def _grab_rsps(socks, rsps, interval, xidmap):
     while r:
         for s in r:
             (rsp, peer) = s.recvfrom(9000)
+            neighutil.refresh_neigh()
             _parse_slp_packet(rsp, peer, rsps, xidmap)
             res = select.select(socks, (), (), interval)
             if not res:
@@ -411,12 +415,11 @@ def rescan(handler):
     known_peers = set([])
     for scanned in scan():
         for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                continue
             if addr in known_peers:
                 break
-            ip = addr[0].partition('%')[0]  # discard scope if present
-            macaddr = neighutil.get_hwaddr(ip)
-            if not macaddr:
-                continue
             known_peers.add(addr)
         else:
             handler(scanned)
@@ -481,10 +484,12 @@ def snoop(handler, protocol=None):
                     ip = peer[0].partition('%')[0]
                     if peer in known_peers:
                         continue
-                    mac = neighutil.get_hwaddr(ip)
-                    if not mac:
+                    if ip not in neighutil.neightable:
+                        neighutil.update_neigh()
+                    if ip not in neighutil.neightable:
                         continue
                     known_peers.add(peer)
+                    mac = neighutil.neightable[ip]
                     if mac in peerbymacaddress:
                         peerbymacaddress[mac]['addresses'].append(peer)
                     else:
@@ -536,12 +541,13 @@ def active_scan(handler, protocol=None):
     known_peers = set([])
     for scanned in scan():
         for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                neighutil.update_neigh()
+            if ip not in neighutil.neightable:
+                continue
             if addr in known_peers:
                 break
-            ip = addr[0].partition('%')[0]  # discard scope if present
-            macaddr = neighutil.get_hwaddr(ip)
-            if not macaddr:
-                continue
             known_peers.add(addr)
         else:
             scanned['protocol'] = protocol

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -61,12 +61,13 @@ def active_scan(handler, protocol=None):
     known_peers = set([])
     for scanned in scan(['urn:dmtf-org:service:redfish-rest:1']):
         for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                neighutil.update_neigh()
+            if ip not in neighutil.neightable:
+                continue
             if addr in known_peers:
                 break
-            ip = addr[0].partition('%')[0]  # discard scope if present
-            hwaddr = neighutil.get_hwaddr(ip)
-            if not hwaddr:
-                continue
             known_peers.add(addr)
         else:
             scanned['protocol'] = protocol
@@ -133,9 +134,11 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                         ip = peer[0].partition('%')[0]
                         if peer in known_peers:
                             continue
-                        mac = neighutil.get_hwaddr(ip)
-                        if not mac:
+                        if ip not in neighutil.neightable:
+                            neighutil.update_neigh()
+                        if ip not in neighutil.neightable:
                             continue
+                        mac = neighutil.neightable[ip]
                         known_peers.add(peer)
                         newmacs.add(mac)
                         if mac in peerbymacaddress:
@@ -204,7 +207,7 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                         msecs = int(currtime * 1000 % 1000)
                                         reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}\r\nCURRTIME: {1}\r\nCURRMSECS: {2}\r\n'.format(node, seconds, msecs)
                                         if '%' in peer[0]:
-                                            iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
+                                            iface = peer[0].split('%', 1)[1]
                                             reply += 'MGTIFACE: {0}\r\n'.format(
                                                 peer[0].split('%', 1)[1])
                                             ncfg = netutil.get_nic_config(
@@ -281,6 +284,7 @@ def _find_service(service, target):
     while r:
         for s in r:
             (rsp, peer) = s.recvfrom(9000)
+            neighutil.refresh_neigh()
             _parse_ssdp(peer, rsp, peerdata)
         timeout = deadline - util.monotonic_time()
         if timeout < 0:
@@ -312,9 +316,11 @@ def _parse_ssdp(peer, rsp, peerdata):
     ip = peer[0].partition('%')[0]
     nid = ip
     mac = None
-    mac = neighutil.get_hwaddr(ip)
-    if mac:
-        nid = mac
+    if ip not in neighutil.neightable:
+        neighutil.update_neigh()
+    if ip in neighutil.neightable:
+        nid = neighutil.neightable[ip]
+        mac = nid
     headlines = rsp.split(b'\r\n')
     try:
         _, code, _ = headlines[0].split(b' ', 2)

confluent_server/confluent/firmwaremanager.py

@@ -138,7 +138,7 @@ class Updater(object):
 
 
 def remove_updates(nodes, tenant, element, type='firmware'):
-    if len(element) < 5 and element[:2] not in (['media', 'uploads'], ['support', 'servicedata']):
+    if len(element) < 5 and element[:2] != ['media', 'uploads']:
         raise exc.InvalidArgumentException()
     upid = element[-1]
     if type == 'firmware':

confluent_server/confluent/httpapi.py

@@ -305,19 +305,9 @@ def _authorize_request(env, operation):
                 # of a CSRF
                 return {'code': 401}
             return ('logout',)
-        if env['HTTP_AUTHORIZATION'].startswith('MultiBasic '):
-            name, passphrase = base64.b64decode(
-                env['HTTP_AUTHORIZATION'].replace('MultiBasic ', '')).split(b':', 1)
-            passphrase = json.loads(passphrase)
-        else:
-            name, passphrase = base64.b64decode(
-                env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(b':', 1)
-        try:
-            authdata = auth.check_user_passphrase(name, passphrase, operation=operation, element=element)
-        except Exception as e:
-            if hasattr(e, 'prompts'):
-                return {'code': 403, 'prompts': e.prompts}
-            raise
+        name, passphrase = base64.b64decode(
+            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(b':', 1)
+        authdata = auth.check_user_passphrase(name, passphrase, operation=operation, element=element)
         if authdata is False:
             return {'code': 403}
         elif not authdata:
@@ -529,14 +519,7 @@ def resourcehandler_backend(env, start_response):
         return
     if authorized['code'] == 403:
         start_response('403 Forbidden', badauth)
-        response = {'result': 'Forbidden'}
-        if 'prompts' in authorized:
-            response['prompts'] = []
-            for prompt in authorized['prompts']:
-                if not isinstance(prompt, str):
-                    prompt = prompt.decode('utf8')
-                response['prompts'].append(prompt)
-        yield json.dumps(response)
+        yield 'Forbidden'
         return
     if authorized['code'] != 200:
         raise Exception("Unrecognized code from auth engine")

confluent_server/confluent/main.py

@@ -59,7 +59,6 @@ import gc
 from greenlet import greenlet
 import sys
 import os
-import glob
 import signal
 import socket
 import time
@@ -210,26 +209,6 @@ def setlimits():
     except Exception:
         pass
 
-def assure_ownership(path):
-    try:
-        if os.getuid() != os.stat(path).st_uid:
-            sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path))
-            sys.exit(1)
-    except OSError as e:
-        if e.errno == 13:
-            sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path))
-            sys.exit(1)
-
-def sanity_check():
-    if os.getuid() == 0:
-        return True
-    assure_ownership('/etc/confluent')
-    assure_ownership('/etc/confluent/cfg')
-    for filename in glob.glob('/etc/confluent/cfg/*'):
-        assure_ownership(filename)
-    assure_ownership('/etc/confluent/privkey.pem')
-    assure_ownership('/etc/confluent/srvcert.pem')
-
 
 def run(args):
     setlimits()
@@ -247,7 +226,6 @@ def run(args):
         sys.stderr.write("Error unlocking credential store\n")
         doexit()
         sys.exit(1)
-    sanity_check()
     try:
         confluentcore.load_plugins()
     except:

confluent_server/confluent/messages.py

@@ -199,8 +199,6 @@ class ConfluentMessage(object):
                 notes.append('Broken because of %s' % val['broken'])
             elif val is not None and 'expression' in val:
                 value = val['expression']
-            elif isinstance(val, dict):
-                value = _htmlify_structure(val)
             if value is None:
                 value = ''
             if val is not None and value == '' and 'isset' in val and val[
@@ -300,7 +298,6 @@ class DeletedResource(ConfluentMessage):
     notnode = True
     def __init__(self, resource):
         self.myargs = [resource]
-        self.desc = 'Delete Resource'
         self.kvpairs = {'deleted': resource}
 
     def strip_node(self, node):
@@ -313,7 +310,6 @@ class CreatedResource(ConfluentMessage):
 
     def __init__(self, resource):
         self.myargs = [resource]
-        self.desc = 'Create Resource'
         self.kvpairs = {'created': resource}
 
     def strip_node(self, node):
@@ -1795,19 +1791,6 @@ class License(ConfluentMessage):
         else:
             self.kvpairs = {name: {'License': kv}}
 
-class GraphicalConsole(ConfluentMessage):
-    readonly = True
-
-    def __init__(self, name=None, mime=None, data=None):
-        self.notnode = name is None
-        self.desc = 'Graphical console launcher'
-
-        kv = []
-        kv.append({'mime': mime, 'data': data})
-        if self.notnode:
-            self.kvpairs = {'Launcher': kv}
-        else:
-            self.kvpairs = {name: {'Launcher': kv}}
 
 class CryptedAttributes(Attributes):
     defaulttype = 'password'

confluent_server/confluent/neighutil.py

@@ -15,88 +15,52 @@
 # limitations under the License.
 
 # A consolidated manage of neighbor table information management.
+# Ultimately, this should use AF_NETLINK, but in the interest of time,
+# use ip neigh for the moment
 
 import confluent.util as util
+import eventlet.green.subprocess as subprocess
 import os
-import eventlet.semaphore as semaphore
-import eventlet.green.socket as socket
-import struct
 
 neightable = {}
 neightime = 0
 
 import re
 
-neighlock = semaphore.Semaphore()
+_validmac = re.compile('..:..:..:..:..:..')
 
-def _update_neigh():
+
+def update_neigh():
     global neightable
     global neightime
-    neightime = os.times()[4]
-    s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, socket.NETLINK_ROUTE)
-    s.bind((0, 0))
-    # RTM_GETNEIGH
-    # nlmsghdr struct: u32 len, u16 type, u16 flags, u32 seq, u32 pid
-    nlhdr = b'\x1c\x00\x00\x00\x1e\x00\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00'
-    # ndmsg struct u8 family u8 pad, u16 pad, s32 ifidx, u16 state, u8 flags, u8 type
-    ndmsg=  b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
-    s.sendall(nlhdr + ndmsg)
     neightable = {}
-    try:
-        while True:
-            pdata = s.recv(65536)
-            v = memoryview(pdata)
-            if struct.unpack('H', v[4:6])[0] == 3:
-                break
-            while len(v):
-                length, typ = struct.unpack('IH', v[:6])
-                if typ == 28:
-                    hlen = struct.calcsize('BIHBB')
-                    _, idx, state, flags, typ = struct.unpack('BIHBB', v[16:16+hlen])
-                    if typ == 1:  # only handle unicast entries
-                        curraddr = None
-                        currip = None
-                        rta = v[16+hlen:length]
-                        while len(rta):
-                            rtalen, rtatyp = struct.unpack('HH', rta[:4])
-                            if rtatyp == 2:  # hwaddr
-                                curraddr = rta[4:rtalen].tobytes()
-                                if len(curraddr) == 20:
-                                    curraddr = curraddr[12:]
-                            elif rtatyp == 1:  # ip address
-                                currip = rta[4:rtalen].tobytes()
-                            rta = rta[rtalen:]
-                            if not rtalen:
-                                break
-                        if curraddr and currip:
-                            neightable[currip] = curraddr
-                v = v[length:]
-    finally:
-        s.close()
-
-
-def get_hwaddr(ipaddr):
-    hwaddr = None
     if os.name == 'nt':
-        return hwaddr
-    if ':' in ipaddr:
-        ipaddr = socket.inet_pton(socket.AF_INET6, ipaddr)
-    elif '.' in ipaddr:
-        ipaddr = socket.inet_pton(socket.AF_INET, ipaddr)
-    with neighlock:
-        updated = False
-        if os.times()[4] > (neightime + 30):
-            _update_neigh()
-            updated = True
-        hwaddr = neightable.get(ipaddr, None)
-        if not hwaddr and not updated:
-            _update_neigh()
-            hwaddr = neightable.get(ipaddr, None)
-    if hwaddr:
-        hwaddr = ':'.join(['{:02x}'.format(x) for x in bytearray(hwaddr)])
-    return hwaddr
+        return
+    ipn = subprocess.Popen(['ip', 'neigh'], stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+    (neighdata, err) = ipn.communicate()
+    neighdata = util.stringify(neighdata)
+    for entry in neighdata.split('\n'):
+        entry = entry.split(' ')
+        if len(entry) < 5 or not entry[4]:
+            continue
+        if entry[0] in ('192.168.0.100', '192.168.70.100', '192.168.70.125'):
+            # Note that these addresses are common static ip addresses
+            # that are hopelessly ambiguous if there are many
+            # so ignore such entries and move on
+            # ideally the system network steers clear of this landmine of
+            # a subnet, but just in case
+            continue
+        if not _validmac.match(entry[4]):
+            continue
+        neightable[entry[0]] = entry[4]
+    neightime = os.times()[4]
 
 
-if __name__ == '__main__':
-    import sys
-    print(repr(get_hwaddr(sys.argv[1])))
+def refresh_neigh():
+    global neightime
+    if os.name == 'nt':
+        return
+    if os.times()[4] > (neightime + 30):
+        update_neigh()

confluent_server/confluent/netutil.py

@@ -39,24 +39,6 @@ def cidr_to_mask(cidr):
     return socket.inet_ntop(
         socket.AF_INET, struct.pack('!I', (2**32 - 1) ^ (2**(32 - cidr) - 1)))
 
-def ipn_on_same_subnet(fam, first, second, prefix):
-    if fam == socket.AF_INET6:
-        if prefix > 64:
-            firstmask = 0xffffffffffffffff
-            secondmask = (2**64-1) ^ (2**(128 - prefix) - 1)
-        else:
-            firstmask = (2**64-1) ^ (2**(64 - prefix) - 1)
-            secondmask = 0
-        first = struct.unpack('!QQ', first)
-        second = struct.unpack('!QQ', second)
-        return ((first[0] & firstmask == second[0] & firstmask)
-            and (first[1] & secondmask == second[1] & secondmask))
-    else:
-        mask = (2**32 - 1) ^ (2**(32 - prefix) - 1)
-        first = struct.unpack('!I', first)[0]
-        second = struct.unpack('!I', second)[0]
-        return (first & mask == second & mask)
-
 def ip_on_same_subnet(first, second, prefix):
     if first.startswith('::ffff:') and '.' in first:
         first = first.replace('::ffff:', '')
@@ -193,220 +175,130 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
         'ipv4_address': None,
         'ipv4_method': None,
         'prefix': None,
-        'ipv6_prefix': None,
-        'ipv6_address': None,
-        'ipv6_method': None,
     }
-    myaddrs = []
+    nets = None
+    needsvrip = False
     if ifidx is not None:
         dhcprequested = False
-        myaddrs = get_my_addresses(ifidx)
-        v4broken = True
-        v6broken = True
-        for addr in myaddrs:
-            if addr[0] == socket.AF_INET:
-                v4broken = False
-            elif addr[0] == socket.AF_INET6:
-                v6broken = False
-        if v4broken:
+        nets = list(_iftonets(ifidx))
+        if not nets:
             cfgdata['ipv4_broken'] = True
-        if v6broken:
-            cfgdata['ipv6_broken'] = True
     if serverip is not None:
-        if '.' in serverip:
-            fam = socket.AF_INET
-        elif ':' in serverip:
-            fam = socket.AF_INET6
-        else:
-            raise ValueError('"{0}" is not a valid ip argument')
-        ipbytes = socket.inet_pton(fam, serverip)
+        needsvrip = True
         dhcprequested = False
-        myaddrs = [x for x in get_my_addresses() if x[1] == ipbytes]
+        nets = list(myiptonets(serverip))
     genericmethod = 'static'
     ipbynodename = None
-    ip6bynodename = None
     try:
-        for addr in socket.getaddrinfo(node, 0, socket.AF_INET, socket.SOCK_DGRAM):
-            ipbynodename = addr[-1][0]
-        
-    except socket.gaierror:
-        pass
-    try:
-        for addr in socket.getaddrinfo(node, 0, socket.AF_INET6, socket.SOCK_DGRAM):
-                ip6bynodename = addr[-1][0]
-    except socket.gaierror:
-        pass
-    if myaddrs:
+        ipbynodename = socket.getaddrinfo(
+            node, 0, socket.AF_INET, socket.SOCK_DGRAM)[0][-1][0]
+    except Exception:
+        ipbynodename = None
+    if nets is not None:
         candgws = []
         candsrvs = []
-        for myaddr in myaddrs:
-            fam, svrip, prefix = myaddr[:3]
-            candsrvs.append((fam, svrip, prefix))
-            if fam == socket.AF_INET:
-                cfgdata['deploy_server'] = socket.inet_ntop(socket.AF_INET, svrip)
-                nver = '4'
-            elif fam == socket.AF_INET6:
-                cfgdata['deploy_server_v6'] = socket.inet_ntop(socket.AF_INET6, svrip)
-                nver = '6'
+        for net in nets:
+            net, prefix, svrip = net
+            candsrvs.append(svrip)
+            cfgdata['deploy_server'] = svrip
             for candidate in cfgbyname:
-                ipmethod = cfgbyname[candidate].get('ipv{}_method'.format(nver), 'static')
+                ipmethod = cfgbyname[candidate].get('ipv4_method', 'static')
                 if ipmethod == 'dhcp':
                     dhcprequested = True
                     continue
                 if ipmethod == 'firmwaredhcp':
                     genericmethod = ipmethod
-                candip = cfgbyname[candidate].get('ipv{}_address'.format(nver), None)
+                candip = cfgbyname[candidate].get('ipv4_address', None)
                 if candip and '/' in candip:
                     candip, candprefix = candip.split('/')
                     if int(candprefix) != prefix:
                         continue
-                candgw = cfgbyname[candidate].get('ipv{}_gateway'.format(nver), None)
+                candgw = cfgbyname[candidate].get('ipv4_gateway', None)
                 if candip:
                     try:
-                        for inf in socket.getaddrinfo(candip, 0, fam, socket.SOCK_STREAM):
-                            candipn = socket.inet_pton(fam, inf[-1][0])
-                        if ipn_on_same_subnet(fam, svrip, candipn, prefix):
-                            cfgdata['ipv{}_address'.format(nver)] = candip
-                            cfgdata['ipv{}_method'.format(nver)] = ipmethod
-                            cfgdata['ipv{}_gateway'.format(nver)] = cfgbyname[candidate].get(
-                                'ipv{}_gateway'.format(nver), None)
-                            if nver == '4':
-                                cfgdata['prefix'] = prefix
-                            else:
-                                cfgdata['ipv{}_prefix'.format(nver)] = prefix
-                            if candip in (ipbynodename, ip6bynodename):
+                        if ip_on_same_subnet(net, candip, prefix):
+                            cfgdata['ipv4_address'] = candip
+                            cfgdata['ipv4_method'] = ipmethod
+                            cfgdata['ipv4_gateway'] = cfgbyname[candidate].get(
+                                'ipv4_gateway', None)
+                            cfgdata['prefix'] = prefix
+                            if ipbynodename and ipbynodename == candip:
                                 cfgdata['matchesnodename'] = True
                             return cfgdata
                     except Exception as e:
                         cfgdata['error_msg'] = "Error trying to evaluate net.*ipv4_address attribute value '{0}' on {1}: {2}".format(candip, node, str(e))
                 elif candgw:
-                    for inf in socket.getaddrinfo(candgw, 0, fam, socket.SOCK_STREAM):
-                        candgwn = socket.inet_pton(fam, inf[-1][0])
-                    if ipn_on_same_subnet(fam, svrip, candgwn, prefix):
-                        candgws.append((fam, candgwn, prefix))
+                    if ip_on_same_subnet(net, candgw, prefix):
+                        candgws.append(candgw)
         if dhcprequested:
             if not cfgdata.get('ipv4_method', None):
                 cfgdata['ipv4_method'] = 'dhcp'
-            if not cfgdata.get('ipv6_method', None):
-                cfgdata['ipv6_method'] = 'dhcp'
             return cfgdata
-        if ipbynodename == None and ip6bynodename == None:
+        if ipbynodename == None:
             return cfgdata
-        for myaddr in myaddrs:
-            fam, svrip, prefix = myaddr[:3]
-            if fam == socket.AF_INET:
-                bynodename = ipbynodename
-                nver = '4'
-            else:
-                bynodename = ip6bynodename
-                nver = '6'
-            if not bynodename:  # node is missing either ipv6 or ipv4, ignore
-                continue
-            ipbynodenamn = socket.inet_pton(fam, bynodename)
-            if ipn_on_same_subnet(fam, svrip, ipbynodenamn, prefix):
+        for net in nets:
+            net, prefix, svrip = net
+            if ip_on_same_subnet(net, ipbynodename, prefix):
                 cfgdata['matchesnodename'] = True
-                cfgdata['ipv{}_address'.format(nver)] = bynodename
-                cfgdata['ipv{}_method'.format(nver)] = genericmethod
-                if nver == '4':
-                    cfgdata['prefix'] = prefix
-                else:
-                    cfgdata['ipv{}_prefix'.format(nver)] = prefix
+                cfgdata['ipv4_address'] = ipbynodename
+                cfgdata['ipv4_method'] = genericmethod
+                cfgdata['prefix'] = prefix
+                break
         for svr in candsrvs:
-            fam, svr, prefix = svr
-            if fam == socket.AF_INET:
-                bynodename = ipbynodename
-            elif fam == socket.AF_INET6:
-                bynodename = ip6bynodename
-            if not bynodename:
-                continue  # ignore undefined family
-            bynodenamn = socket.inet_pton(fam, bynodename)
-            if ipn_on_same_subnet(fam, svr, bynodenamn, prefix):
-                svrname = socket.inet_ntop(fam, svr)
-                if fam == socket.AF_INET:
-                    cfgdata['deploy_server'] = svrname
-                else:
-                    cfgdata['deploy_server_v6'] = svrname
+            if ip_on_same_subnet(svr, ipbynodename, prefix):
+                cfgdata['deploy_server'] = svr
+                break
         for gw in candgws:
-            fam, candgwn, prefix = gw
-            if fam == socket.AF_INET:
-                nver = '4'
-                bynodename = ipbynodename
-            elif fam == socket.AF_INET6:
-                nver = '6'
-                bynodename = ip6bynodename
-            bynodenamn = socket.inet_pton(fam, bynodename)
-            if ipn_on_same_subnet(fam, candgwn, bynodenamn, prefix):
-                cfgdata['ipv{}_gateway'.format(nver)] = socket.inet_ntop(fam, candgwn)
+            if ip_on_same_subnet(gw, ipbynodename, prefix):
+                cfgdata['ipv4_gateway'] = gw
+                break
         return cfgdata
     if ip is not None:
-        for prefixinfo in get_prefix_len_for_ip(ip):
-            fam, prefix = prefixinfo
-            if fam == socket.AF_INET:
-                cfgdata['prefix'] = prefix
-                nver = '4'
-            else:
-                nver = '6'
-                cfgdata['ipv{}_prefix'.format(nver)] = prefix
-            for setting in nodenetattribs:
-                if 'ipv{}_gateway'.format(nver) not in setting:
-                    continue
-                gw = nodenetattribs[setting].get('value', None)
-                if gw is None or not gw:
-                    continue
-                gwn = socket.inet_pton(fam, gw)
-                ipn = socket.inet_pton(fam, ip)
-                if ipn_on_same_subnet(fam, ipn, gwn, prefix):
-                    cfgdata['ipv{}_gateway'.format(nver)] = gw
-                    break
+        prefixlen = get_prefix_len_for_ip(ip)
+        cfgdata['prefix'] = prefixlen
+        for setting in nodenetattribs:
+            if 'ipv4_gateway' not in setting:
+                continue
+            gw = nodenetattribs[setting].get('value', None)
+            if gw is None or not gw:
+                continue
+            if ip_on_same_subnet(ip, gw, prefixlen):
+                cfgdata['ipv4_gateway'] = gw
+                break
     return cfgdata
 
-nlhdrsz = struct.calcsize('IHHII')
-ifaddrsz = struct.calcsize('BBBBI')
-
-def get_my_addresses(idx=0, family=0):
-    # RTM_GETADDR = 22
-    # nlmsghdr struct: u32 len, u16 type, u16 flags, u32 seq, u32 pid
-    nlhdr = struct.pack('IHHII', nlhdrsz + ifaddrsz, 22, 0x301, 0, 0)
-    # ifaddrmsg struct: u8 family, u8 prefixlen, u8 flags, u8 scope, u32 index
-    ifaddrmsg = struct.pack('BBBBI', family, 0, 0, 0, idx)
-    s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, socket.NETLINK_ROUTE)
-    s.bind((0, 0))
-    s.sendall(nlhdr + ifaddrmsg)
-    addrs = []
-    while True:
-        pdata = s.recv(65536)
-        v = memoryview(pdata)
-        if struct.unpack('H', v[4:6])[0] == 3:  # netlink done message
-            break
-        while len(v):
-            length, typ = struct.unpack('IH', v[:6])
-            if typ == 20:
-                fam, plen, _, scope, ridx = struct.unpack('BBBBI', v[nlhdrsz:nlhdrsz+ifaddrsz])
-                if (ridx == idx or not idx) and scope == 0:
-                    rta = v[nlhdrsz+ifaddrsz:length]
-                    while len(rta):
-                        rtalen, rtatyp = struct.unpack('HH', rta[:4])
-                        if rtatyp == 1:
-                            addrs.append((fam, rta[4:rtalen].tobytes(), plen, ridx))
-                        if not rtalen:
-                            break
-                        rta = rta[rtalen:]
-            v = v[length:]
-    return addrs
-
 
 def get_prefix_len_for_ip(ip):
-    myaddrs = get_my_addresses()
-    found = False
-    for inf in socket.getaddrinfo(ip, 0, 0, socket.SOCK_DGRAM):
-        for myaddr in myaddrs:
-            if inf[0] != myaddr[0]:
-                continue
-            if ipn_on_same_subnet(myaddr[0], myaddr[1], socket.inet_pton(myaddr[0], inf[-1][0]), myaddr[2]):
-                found = True
-                yield (myaddr[0], myaddr[2])
-    if not found:
-        raise exc.NotImplementedException("Non local addresses not supported")
+    # for now, we'll use the system route table
+    # later may provide for configuration lookup to override the route
+    # table
+    ip = getaddrinfo(ip, 0, socket.AF_INET)[0][-1][0]
+    try:
+        ipn = socket.inet_aton(ip)
+    except socket.error:  # For now, assume 64 for ipv6
+        return 64
+    # It comes out big endian, regardless of host arch
+    ipn = struct.unpack('>I', ipn)[0]
+    rf = open('/proc/net/route')
+    ri = rf.read()
+    rf.close()
+    ri = ri.split('\n')[1:]
+    for rl in ri:
+        if not rl:
+            continue
+        rd = rl.split('\t')
+        if rd[1] == '00000000':  # default gateway, not useful for this
+            continue
+        # don't have big endian to look at, assume that it is host endian
+        maskn = struct.unpack('I', struct.pack('>I', int(rd[7], 16)))[0]
+        netn = struct.unpack('I', struct.pack('>I', int(rd[1], 16)))[0]
+        if ipn & maskn == netn:
+            nbits = 0
+            while maskn:
+                nbits += 1
+                maskn = maskn << 1 & 0xffffffff
+            return nbits
+    raise exc.NotImplementedException("Non local addresses not supported")
 
 def addresses_match(addr1, addr2):
     """Check two network addresses for similarity

confluent_server/confluent/noderange.py

@@ -100,12 +100,11 @@ class NodeRange(object):
     :param config: Config manager object to use to vet elements
     """
 
-    def __init__(self, noderange, config=None, purenumeric=False):
+    def __init__(self, noderange, config=None):
         global lastnoderange
         self.beginpage = None
         self.endpage = None
         self.cfm = config
-        self.purenumeric = purenumeric
         try:
             elements = _parser.parseString("(" + noderange + ")", parseAll=True).asList()[0]
         except pp.ParseException as pe:
@@ -160,27 +159,16 @@ class NodeRange(object):
         return set([atom])
 
     def expandrange(self, seqrange, delimiter):
-        increment = 1
-        if self.purenumeric:
-            seqranges = seqrange.replace('-', ':').replace('..', ':')
-            pieces = seqranges.split(':')
-            if len(pieces) > 3:
-                raise Exception("Invalid numeric sequence")
-            leftbits = [pieces[0]]
-            rightbits = [pieces[1]]
-            if len(pieces) == 3:
-                increment = int(pieces[2])
-        else:
-            pieces = seqrange.split(delimiter)
-            if len(pieces) % 2 != 0:
-                return self.failorreturn(seqrange)
-            halflen = len(pieces) // 2
-            left = delimiter.join(pieces[:halflen])
-            right = delimiter.join(pieces[halflen:])
-            leftbits = _numextractor.parseString(left).asList()
-            rightbits = _numextractor.parseString(right).asList()
-            if len(leftbits) != len(rightbits):
-                return self.failorreturn(seqrange)
+        pieces = seqrange.split(delimiter)
+        if len(pieces) % 2 != 0:
+            return self.failorreturn(seqrange)
+        halflen = len(pieces) // 2
+        left = delimiter.join(pieces[:halflen])
+        right = delimiter.join(pieces[halflen:])
+        leftbits = _numextractor.parseString(left).asList()
+        rightbits = _numextractor.parseString(right).asList()
+        if len(leftbits) != len(rightbits):
+            return self.failorreturn(seqrange)
         finalfmt = ''
         iterators = []
         for idx in range(len(leftbits)):
@@ -206,7 +194,7 @@ class NodeRange(object):
                 else:  # differently padded, but same number...
                     return self.failorreturn(seqrange)
                 numformat = '{0:0%d}' % width
-                for num in range(minnum, maxnum, increment):
+                for num in range(minnum, maxnum):
                     curseq.append(numformat.format(num))
         results = set([])
         for combo in itertools.product(*iterators):
@@ -242,7 +230,7 @@ class NodeRange(object):
         for idx in range(len(element)):
             if element[idx][0] == '[':
                 nodes = set([])
-                for numeric in NodeRange(element[idx][1:-1], purenumeric=True).nodes:
+                for numeric in NodeRange(element[idx][1:-1]).nodes:
                     nodes |= self._expandstring(
                         [prefix + numeric] + element[idx + 1:])
                 return nodes

confluent_server/confluent/osimage.py

@@ -18,12 +18,11 @@ import yaml
 COPY = 1
 EXTRACT = 2
 READFILES = set([
-    '.disk/info',
+    'README.diskdefines',
     'media.1/products',
     'media.2/products',
     '.DISCINFO',
     '.discinfo',
-    'zipl.prm',
 ])
 
 HEADERSUMS = set([b'\x85\xeddW\x86\xc5\xbdhx\xbe\x81\x18X\x1e\xb4O\x14\x9d\x11\xb7C8\x9b\x97R\x0c-\xb8Ht\xcb\xb3'])
@@ -336,24 +335,26 @@ def check_esxi(isoinfo):
         }
 
 def check_ubuntu(isoinfo):
-    if '.disk/info' not in isoinfo[1]:
+    if 'README.diskdefines' not in isoinfo[1]:
         return None
     arch = None
     variant = None
     ver = None
-    diskdefs = isoinfo[1]['.disk/info']
+    diskdefs = isoinfo[1]['README.diskdefines']
     for info in diskdefs.split(b'\n'):
         if not info:
             continue
-        info = info.split(b' ')
-        name = info[0].strip()
-        ver = info[1].strip()
-        arch = info[-2].strip()
-        if name != b'Ubuntu-Server':
-            return None
-        if arch == b'amd64':
-            arch = b'x86_64'
-    if ver:
+        _, key, val = info.split(b' ', 2)
+        val = val.strip()
+        if key == b'ARCH':
+            arch = val
+            if arch == b'amd64':
+                arch = b'x86_64'
+        elif key == b'DISKNAME':
+            variant, ver, _ = val.split(b' ', 2)
+            if variant != b'Ubuntu-Server':
+                return None
+    if variant:
         if not isinstance(ver, str):
             ver = ver.decode('utf8')
         if not isinstance(arch, str):
@@ -423,37 +424,6 @@ def _priv_check_oraclelinux(isoinfo):
     return {'name': 'oraclelinux-{0}-{1}'.format(ver, arch), 'method': EXTRACT,
             'category': 'el{0}'.format(major)}
 
-
-def fixup_coreos(targpath):
-    # the efi boot image holds content that the init script would want
-    # to mcopy, but the boot sector is malformed usually, so change it to 1
-    # sector per track
-    if os.path.exists(targpath + '/images/efiboot.img'):
-        with open(targpath + '/images/efiboot.img', 'rb+') as bootimg:
-            bootimg.seek(0x18)
-            if bootimg.read != b'\x00\x00':
-                bootimg.seek(0x18)
-                bootimg.write(b'\x01')
-
-
-def check_coreos(isoinfo):
-    arch = 'x86_64'  # TODO: would check magic of vmlinuz to see which arch
-    if 'zipl.prm' in isoinfo[1]:
-        prodinfo = isoinfo[1]['zipl.prm']
-        if not isinstance(prodinfo, str):
-            prodinfo = prodinfo.decode('utf8')
-        for inf in prodinfo.split():
-            if inf.startswith('coreos.liveiso=rhcos-'):
-                ver = inf.split('-')[1]
-                return {'name': 'rhcos-{0}-{1}'.format(ver, arch),
-                        'method': EXTRACT, 'category': 'coreos'}
-            elif inf.startswith('coreos.liveiso=fedore-coreos-'):
-                ver = inf.split('-')[2]
-                return {'name': 'fedoracoreos-{0}-{1}'.format(ver, arch),
-                        'method': EXTRACT, 'category': 'coreos'}
-
-
-
 def check_rhel(isoinfo):
     ver = None
     arch = None
@@ -532,7 +502,7 @@ def fingerprint(archive):
             if fun.startswith('check_'):
                 name = globals()[fun](isoinfo)
                 if name:
-                    return name, isoinfo[0], fun.replace('check_', '')
+                    return name, isoinfo[0]
         return None
     else:
         sum = hashlib.sha256(header)
@@ -544,7 +514,7 @@ def fingerprint(archive):
                 chunk = archive.read(32768)
             imginfo = HASHPRINTS.get(sum.hexdigest(), None)
             if imginfo:
-                return imginfo, None, None
+                return imginfo, None
 
 
 def import_image(filename, callback, backend=False, mfd=None):
@@ -555,7 +525,7 @@ def import_image(filename, callback, backend=False, mfd=None):
     identity = fingerprint(archive)
     if not identity:
         return -1
-    identity, imginfo, funname = identity
+    identity, imginfo = identity
     targpath = identity['name']
     distpath = '/var/lib/confluent/distributions/' + targpath
     if identity.get('subname', None):
@@ -596,8 +566,6 @@ def import_image(filename, callback, backend=False, mfd=None):
         del identity['subname']
     with open(distpath + '/distinfo.yaml', 'w') as distinfo:
         distinfo.write(yaml.dump(identity, default_flow_style=False))
-    if 'fixup_{0}'.format(funname) in globals():
-        globals()['fixup_{0}'.format(funname)](targpath)
     callback({'progress': 1.0})
     sys.stdout.write('\n')
 
@@ -607,10 +575,10 @@ def printit(info):
 
 
 def list_distros():
-    return sorted(os.listdir('/var/lib/confluent/distributions'))
+    return os.listdir('/var/lib/confluent/distributions')
 
 def list_profiles():
-    return sorted(os.listdir('/var/lib/confluent/public/os/'))
+    return os.listdir('/var/lib/confluent/public/os/')
 
 def get_profile_label(profile):
     with open('/var/lib/confluent/public/os/{0}/profile.yaml') as metadata:
@@ -670,8 +638,6 @@ class MediaImporter(object):
 
     def __init__(self, media, cfm=None):
         self.worker = None
-        if not os.path.exists('/var/lib/confluent/public'):
-            raise Exception('`osdeploy initialize` must be executed before importing any media')
         self.profiles = []
         medfile = None
         if cfm and media in cfm.clientfiles:
@@ -682,7 +648,7 @@ class MediaImporter(object):
         if not identity:
             raise exc.InvalidArgumentException('Unsupported Media')
         self.percent = 0.0
-        identity, _, _ = identity
+        identity, _ = identity
         self.phase = 'copying'
         if not identity:
             raise Exception('Unrecognized OS Media')