Have SSDP ignore multicast disabled interfaces

A frequent scenario is to 'refresh' ssh configuration toward the
end of:
-changing trust nodes
-Adding a collective member
-Repairing a broken configuration
-As part of 'confluent-ifying' a node that wasn't confluent deployed

confluent_client/bin/nodeattrib

@@ -105,7 +105,10 @@ if len(args) > 1:
             requestargs=args[1:]
     except:
         pass
-
+elif options.clear or options.environment or options.prompt:
+    sys.stderr.write('Attribute names required with specified options\n')
+    argparser.print_help()
+    exitcode = 400
 if exitcode != 0:
     sys.exit(exitcode)
 

confluent_client/bin/nodefirmware

@@ -88,9 +88,9 @@ client.check_globbing(noderange)
 
 def get_update_progress(session, url):
     for res in session.read(url):
-        status = res['phase']
-        percent = res['progress']
-        detail = res['detail']
+        status = res.get('phase', 'error')
+        percent = res.get('progress', None)
+        detail = res.get('detail', repr(res)),
         if status == 'error':
             text = 'error!'
         else:

confluent_client/bin/noderun

@@ -131,8 +131,14 @@ def run():
 
 
 def run_cmdv(node, cmdv, all, pipedesc):
-    nopen = subprocess.Popen(
-        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    try:
+        nopen = subprocess.Popen(
+            cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError as e:
+        if e.errno == 2:
+            sys.stderr.write('{0}: Unable to find local executable file "{1}"'.format(node, cmdv[0]))
+            return
+        raise
     pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
                               'type': 'stdout'}
     pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,

confluent_client/confluent/client.py

@@ -42,9 +42,9 @@ _attraliases = {
 }
 
 try:
-    input = raw_input
+    getinput = raw_input
 except NameError:
-    pass
+    getinput = input
 
 
 class NestedDict(dict):
@@ -284,9 +284,9 @@ class Command(object):
                 nodename = list(self.read(
                     '/noderange/{0}/nodes/'.format(noderange)))[0].get('item', {}).get('href', None)
                 nodename = nodename[:-1]
-                p = input('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
+                p = getinput('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
             else:
-                p = input('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
+                p = getinput('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
             if p.lower() != 'y':
                 sys.stderr.write('Aborting at user request\n')
                 sys.exit(1)
@@ -401,7 +401,7 @@ class Command(object):
                 if fingerprint == khf[hostid]:
                     return
                 else:
-                    replace = input(
+                    replace = getinput(
                         "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")

confluent_client/confluent_client.spec.tmpl

@@ -27,15 +27,23 @@ a confluent server.
 %if "%{dist}" == ".el8"
 python3 setup.py build
 %else
+%if "%{dist}" == ".el9"
+python3 setup.py build
+%else
 python2 setup.py build
 %endif
+%endif
 
 %install
 %if "%{dist}" == ".el8"
 python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
 %else
+%if "%{dist}" == ".el9"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%else
 python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
 %endif
+%endif
 
 
 %clean

confluent_client/doc/man/nodeshell.ronn

@@ -17,6 +17,9 @@ as stderr, unlike psh which combines all stdout and stderr into stdout.
 * `-c COUNT`, `-f COUNT`, `--count=COUNT`
   Specify the maximum number of instances to run concurrently
 
+* `-l LOGINNAME`, `--loginname=LOGINNAME`
+  Username to use when connecting, defaults to current user.
+
 * `-m MAXNODES`, `--maxnodes=MAXNODES`
   Specify a maximum number of nodes to run remote ssh command to, prompting
   if over the threshold

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -18,12 +18,15 @@ import time
 class InvalidApiKey(Exception):
     pass
 
+def msg_align(len):
+    return (len + 3) & ~3
+
 cryptname = ctypes.util.find_library('crypt')
 if not cryptname:
-    if os.path.exists('/usr/lib64/libcrypt.so.1'):
-        cryptname = 'libcrypt.so.1'
-    elif os.path.exists('/usr/lib64/libcrypt.so.2'):
+    if os.path.exists('/lib64/libcrypt.so.2') or os.path.exists('/usr/lib64/libcrypt.so.2'):
         cryptname = 'libcrypt.so.2'
+    else:
+        cryptname = 'libcrypt.so.1'
 c_libcrypt = ctypes.CDLL(cryptname)
 c_crypt = c_libcrypt.crypt
 c_crypt.argtypes = (ctypes.c_char_p, ctypes.c_char_p)
@@ -59,8 +62,8 @@ def get_my_addresses():
                             break
                         if rtatyp == 1:
                             addrs.append((fam, rta[4:rtalen], plen, ridx))
-                        rta = rta[rtalen:]
-            v = v[length:]
+                        rta = rta[msg_align(rtalen):]
+            v = v[msg_align(length):]
     return addrs
 
 
@@ -93,14 +96,20 @@ def scan_confluents():
         if addr[0] == socket.AF_INET6:
             if addr[-1] in doneidxs:
                 continue
-            s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addr[-1])
+            try:
+                s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addr[-1])
+            except TypeError:
+                s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addr[-1].tobytes())
             try:
                 s6.sendto(msg, ('ff02::c', 1900))
             except OSError:
                 pass
             doneidxs.add(addr[-1])
         elif addr[0] == socket.AF_INET:
-            s4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, addr[1])
+            try:
+                s4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, addr[1])
+            except TypeError:
+                s4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, addr[1].tobytes())
             try:
                 s4.sendto(msg, ('239.255.255.250', 1900))
             except OSError:
@@ -138,7 +147,7 @@ def get_net_apikey(nodename, mgr):
     newpass = newpass.encode('utf8')
     salt = salt.encode('utf8')
     crypted = c_crypt(newpass, salt)
-    for addrinfo in socket.getaddrinfo(mgr, 13001, type=socket.SOCK_STREAM):
+    for addrinfo in socket.getaddrinfo(mgr, 13001, 0, socket.SOCK_STREAM):
         try:
             clisock = socket.socket(addrinfo[0], addrinfo[1])
             clisock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -174,7 +183,7 @@ def get_net_apikey(nodename, mgr):
     return ''
 
 
-def get_apikey(nodename, hosts):
+def get_apikey(nodename, hosts, errout=None):
     apikey = ""
     if os.path.exists('/etc/confluent/confluent.apikey'):
         apikey = open('/etc/confluent/confluent.apikey').read().strip()
@@ -198,8 +207,10 @@ def get_apikey(nodename, hosts):
                 if apikey:
                     break
         if not apikey:
-            sys.stderr.write(
-                "Failed getting API token, check deployment.apiarmed attribute on {}\n".format(nodename))
+            errmsg = "Failed getting API token, check deployment.apiarmed attribute on {}\n".format(nodename)
+            sys.stderr.write(errmsg)
+            if errout:
+                errout.write(errmsg)
             time.sleep(10)
     with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
         apiout.write(apikey)
@@ -208,7 +219,11 @@ def get_apikey(nodename, hosts):
     return apikey
 
 class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, usejson=False, port=443, host=None):
+    def __init__(self, usejson=False, port=443, host=None, errout=None):
+        self.errout = None
+        if errout:
+            self.errout = open(errout, 'w')
+            self.errout.flush()
         self.stdheaders = {}
         mgtiface = None
         if usejson:
@@ -251,7 +266,7 @@ class HTTPSClient(client.HTTPConnection, object):
                 for line in info:
                     if line.startswith('deploy_server: ') or line.startswith('deploy_server_v6: '):
                         self.hosts.append(line.split(': ', 1)[1])
-        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts)
+        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
         self.port = port
@@ -319,7 +334,15 @@ class HTTPSClient(client.HTTPConnection, object):
             self.stdheaders['Host'] = '{0}'.format(host)
         ctx.verify_mode = ssl.CERT_REQUIRED
         ctx.check_hostname = True
-        self.sock = ctx.wrap_socket(psock, server_hostname=host)
+        try:
+            self.sock = ctx.wrap_socket(psock, server_hostname=host)
+        except ssl.SSLError:
+            errmsg = 'Error validating certificate on deployer (try `osdeploy initialize -t` on the deployment server {0})\n'.format(host)
+            sys.stderr.write(errmsg)
+            if self.errout:
+                self.errout.write(errmsg)
+                self.errout.flush()
+            sys.exit(1)
 
     def grab_url(self, url, data=None, returnrsp=False):
         return self.grab_url_with_status(url, data, returnrsp)[1]
@@ -345,7 +368,7 @@ class HTTPSClient(client.HTTPConnection, object):
                 with open('/etc/confluent/confluent.apikey', 'w+') as akfile:
                     akfile.write('')
                 self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(
-                    self.node, [self.host])
+                    self.node, [self.host], errout=self.errout)
         raise Exception(rsp.read())
 
 if __name__ == '__main__':
@@ -374,22 +397,28 @@ if __name__ == '__main__':
         data = sys.argv.pop(data)
     except ValueError:
         data = None
+    try:
+        errout = sys.argv.index('-e')
+        sys.argv.pop(errout)
+        errout = sys.argv.pop(errout)
+    except ValueError:
+        errout = None
+    if len(sys.argv) > 2 and os.path.exists(sys.argv[-1]):
+        data = open(sys.argv[-1]).read()
     if outbin:
         with open(outbin, 'ab+') as outf:
-            reader = HTTPSClient(usejson=usejson).grab_url(
+            reader = HTTPSClient(usejson=usejson, errout=errout).grab_url(
                 sys.argv[1], data, returnrsp=True)
             chunk = reader.read(16384)
             while chunk:
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
-    if len(sys.argv) > 2 and os.path.exists(sys.argv[-1]):
-        data = open(sys.argv[-1]).read()
     if waitfor:
-        client = HTTPSClient(usejson)
+        client = HTTPSClient(usejson, errout=errout)
         status = 201
         while status != waitfor:
             status, rsp = client.grab_url_with_status(sys.argv[1], data)
             sys.stdout.write(rsp.decode())
     else:
-        sys.stdout.write(HTTPSClient(usejson).grab_url(sys.argv[1], data).decode())
+        sys.stdout.write(HTTPSClient(usejson, errout=errout).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/common/profile/scripts/confignet

@@ -8,11 +8,17 @@ import sys
 import time
 import shlex
 import subprocess
-from importlib.machinery import SourceFileLoader
 try:
-    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
-except FileNotFoundError:
-    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+    from importlib.machinery import SourceFileLoader
+    def load_source(mod, path):
+        return SourceFileLoader(mod, path).load_module()
+except ImportError:
+    from imp import load_source
+
+try:
+    apiclient = load_source('apiclient', '/opt/confluent/bin/apiclient')
+except IOError:
+    apiclient = load_source('apiclient', '/etc/confluent/apiclient')
 
 def add_lla(iface, mac):
     pieces = mac.split(':')
@@ -311,7 +317,7 @@ if __name__ == '__main__':
             myname = myname[0]
             myname = socket.inet_pton(socket.AF_INET, myname)
             for addr in myaddrs:
-                if myname == bytes(addr[1]):
+                if myname == addr[1].tobytes():
                     curridx = addr[-1]
         if curridx in doneidxs:
             continue

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -44,7 +44,7 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos el9; do
     mv ../addons.cpio .
     cd ..
 done
-for os in el8 suse15 el9 ubuntu20.04; do
+for os in el7 el8 suse15 el9 ubuntu20.04; do
     mkdir ${os}disklessout
     cd ${os}disklessout
     if [ -d ../${os}bin ]; then

confluent_osdeploy/coreos/initramfs/usr/lib/systemd/system/confluent-rootfs.service

@@ -6,6 +6,7 @@ After=basic.target
 # Network is enabled here
 After=dracut-initqueue.service
 Before=ignition-setup-user.service
+Before=coreos-ignition-setup-user.service
 Before=coreos-livepxe-rootfs.service
 
 

confluent_osdeploy/el7-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -0,0 +1,245 @@
+get_remote_apikey() {
+    while [ -z "$confluent_apikey" ]; do
+        /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
+        if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
+            # we don't support remote sealed api keys anymore
+            echo > /etc/confluent/confluent.apikey
+        fi
+        confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+        if [ -z "$confluent_apikey" ]; then
+            echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
+            sleep 10
+        elif [ -c /dev/tpmrm0 -a -x /usr/bin/tpm2_startauthsession ]; then
+            tmpdir=$(mktemp -d)
+            cd $tmpdir
+            tpm2_startauthsession --session=session.ctx
+            tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+            tpm2_createprimary -G ecc -Q --key-context=prim.ctx
+            (echo -n "CONFLUENT_APIKEY:";cat /etc/confluent/confluent.apikey) | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
+            tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
+            tpm2_evictcontrol -Q -c data.ctx
+            tpm2_flushcontext session.ctx
+            cd - > /dev/null
+            rm -rf $tmpdir
+        fi
+    done
+}
+root=1
+rootok=1
+netroot=confluent
+clear
+mkdir -p /etc/ssh
+mkdir -p /var/tmp/
+mkdir -p /var/empty/sshd
+mkdir -p /etc/confluent
+sed -i '/^root:x/d' /etc/passwd
+echo root:x:0:0::/:/bin/bash >> /etc/passwd
+echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
+
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    autocons=${autocons##*/}
+    echo "Automatic console configured for $autocons"
+fi
+echo "Initializing confluent diskless environment"
+echo -n "udevd: "
+/usr/lib/systemd/systemd-udevd --daemon
+echo -n "Loading drivers..."
+udevadm trigger
+udevadm trigger --type=devices --action=add
+udevadm settle
+modprobe ib_ipoib
+modprobe ib_umad
+modprobe hfi1
+modprobe mlx5_ib
+echo "done"
+cat > /etc/ssh/sshd_config << EOF
+Port 2222
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
+PermitRootLogin yes
+AuthorizedKeysFile      .ssh/authorized_keys
+EOF
+mkdir /root/.ssh
+mkdir /.ssh
+cat /ssh/*pubkey > /root/.ssh/authorized_keys 2>/dev/null
+cp /root/.ssh/authorized_keys /.ssh/
+cat /tls/*.pem > /etc/confluent/ca.pem
+mkdir -p /etc/pki/tls/certs
+cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
+TRIES=0
+oldumask=$(umask)
+umask 0077
+tpmdir=$(mktemp -d)
+cd $tpmdir
+lasthdl=""
+if [ -c /dev/tpmrm0 -a -x /usr/bin/tpm2_getcap ]; then
+    for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
+        tpm2_startauthsession --policy-session --session=session.ctx
+        tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+        unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
+        tpm2_flushcontext session.ctx
+        if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
+            confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
+            echo $confluent_apikey > /etc/confluent/confluent.apikey
+            if [ -n "$lasthdl" ]; then
+                tpm2_evictcontrol -c $lasthdl
+            fi
+            lasthdl=$hdl
+        fi
+    done
+fi
+cd - > /dev/null
+rm -rf $tpmdir
+touch /etc/confluent/confluent.info
+cd /sys/class/net
+echo -n "Scanning for network configuration..."
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
+    TRIES=$((TRIES + 1))
+    for i in *; do
+        ip link set $i up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+hostname $nodename
+confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1 | awk -F': ' '{print $2}' | awk -F'|' '{print $1}')
+if [ -z "$confluent_mgr" ]; then
+    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
+fi
+if [[ $confluent_mgr == *%* ]]; then
+    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+
+ready=0
+while [ $ready = "0" ]; do
+    get_remote_apikey
+    if [[ $confluent_mgr == *:* ]]; then
+        confluent_mgr="[${confluent_mgr/\%/%25}]"
+    fi
+    tmperr=$(mktemp)
+    curl -gsSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
+    if grep 401 $tmperr > /dev/null; then
+        confluent_apikey=""
+        if [ -n "$lasthdl" ]; then
+            tpm2_evictcontrol -c $lasthdl
+        fi
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    else
+        ready=1
+    fi
+    rm $tmperr
+done
+if [ ! -z "$autocons" ] && grep textconsole: true /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
+if [ -c /dev/tpmrm0 -a -x /usr/bin/tpm2_pcrextend ]; then
+    tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
+fi
+umask $oldumask
+autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo -n "Attempting to use dhcp to bring up $ifname..."
+    dhclient $ifname
+    echo "Complete:"
+    ip addr show dev $ifname
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#prefix: }
+    echo "Setting up $ifname as static at $v4addr/$v4nm"
+    ip addr add dev $ifname $v4addr/$v4nm
+    if [ ! -z "$v4gw" ]; then
+        ip route add default via $v4gw
+    fi
+    mkdir -p /run/NetworkManager/system-connections
+    cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[connection]
+EOC
+    echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+type=ethernet
+autoconnect-retries=1
+EOC
+    echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+multi-connect=1
+permissions=
+wait-device-timeout=60000
+
+[ethernet]
+mac-address-blacklist=
+
+[ipv4]
+EOC
+    echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v4gw" ]; then
+        echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+may-fail=false
+method=manual
+
+[ipv6]
+addr-gen-mode=eui64
+method=auto
+
+[proxy]
+EOC
+fi
+chmod 600 /run/NetworkManager/system-connections/*.nmconnection
+echo -n "Initializing ssh..."
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    curl -gsf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_urls=""
+for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed -e s/%/%25/); do
+    if [[ $addr == *:* ]]; then
+        confluent_urls="$confluent_urls $confluent_proto://[$addr]/confluent-public/os/$confluent_profile/rootimg.sfs"
+    else
+        confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    fi
+done
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+mkdir -p /etc/confluent
+curl -gsf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+source_remote imageboot.sh

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+exec >> /var/log/confluent/confluent-firstboot.log
+exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
+tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
+logshowpid=$!
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+
+if [ ! -f /etc/confluent/firstboot.ran ]; then
+    touch /etc/confluent/firstboot.ran
+
+    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+
+    run_remote firstboot.custom
+    # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+    run_remote_parts firstboot.d
+
+    # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+    run_remote_config firstboot.d
+fi
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service
+rm /etc/confluent/firstboot.ran
+kill $logshowpid

confluent_osdeploy/el7-diskless/profiles/default/scripts/functions

@@ -0,0 +1,196 @@
+#!/bin/bash
+function test_mgr() {
+    if curl -s https://${1}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,93 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py

@@ -0,0 +1,376 @@
+#!/usr/bin/python3
+import glob
+import json
+import os
+import re
+import time
+import shutil
+import socket
+import stat
+import struct
+import sys
+import subprocess
+
+def get_next_part_meta(img, imgsize):
+    if img.tell() == imgsize:
+        return None
+    pathlen = struct.unpack('!H', img.read(2))[0]
+    mountpoint = img.read(pathlen).decode('utf8')
+    jsonlen = struct.unpack('!I', img.read(4))[0]
+    metadata = json.loads(img.read(jsonlen).decode('utf8'))
+    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip filesystem type
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip orig devname (redundant with json)
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip padding
+    nextlen = struct.unpack('!Q', img.read(8))[0]
+    img.seek(nextlen, 1)  # go to next section
+    return metadata
+
+def get_multipart_image_meta(img):
+    img.seek(0, 2)
+    imgsize = img.tell()
+    img.seek(16)
+    seekamt = img.read(1)
+    img.seek(struct.unpack('B', seekamt)[0], 1)
+    partinfo = get_next_part_meta(img, imgsize)
+    while partinfo:
+        yield partinfo
+        partinfo = get_next_part_meta(img, imgsize)
+
+def get_image_metadata(imgpath):
+    with open(imgpath, 'rb') as img:
+        header = img.read(16)
+        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
+            for md in get_multipart_image_meta(img):
+                yield md
+        else:
+            raise Exception('Installation from single part image not supported')
+
+class PartedRunner():
+    def __init__(self, disk):
+        self.disk = disk
+
+    def run(self, command):
+        command = command.split()
+        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
+        return subprocess.check_output(command).decode('utf8')
+
+def fixup(rootdir, vols):
+    devbymount = {}
+    for vol in vols:
+        devbymount[vol['mount']] = vol['targetdisk']
+    fstabfile = os.path.join(rootdir, 'etc/fstab')
+    with open(fstabfile) as tfile:
+        fstab = tfile.read().split('\n')
+    while not fstab[0]:
+        fstab = fstab[1:]
+    if os.path.exists(os.path.join(rootdir, '.autorelabel')):
+        os.unlink(os.path.join(rootdir, '.autorelabel'))
+    with open(fstabfile, 'w') as tfile:
+        for tab in fstab:
+            entry = tab.split()
+            if tab.startswith('#ORIGFSTAB#'):
+                if entry[1] in devbymount:
+                    targetdev = devbymount[entry[1]]
+                    if targetdev.startswith('/dev/localstorage/'):
+                        entry[0] = targetdev
+                    else:
+                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
+                        uuid = uuid.strip()
+                        entry[0] = 'UUID={}'.format(uuid)
+                elif entry[2] == 'swap':
+                    entry[0] = '/dev/mapper/localstorage-swap'
+                entry[0] = entry[0].ljust(42)
+                entry[1] = entry[1].ljust(16)
+                entry[3] = entry[3].ljust(28)
+                tab = '\t'.join(entry)
+            tfile.write(tab + '\n')
+    with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout:
+        nameout.write(socket.gethostname() + '\n')
+    selinuxconfig = os.path.join(rootdir, 'etc/selinux/config')
+    policy = None
+    if os.path.exists(selinuxconfig):
+        with open(selinuxconfig) as cfgin:
+            sec = cfgin.read().split('\n')
+            for l in sec:
+                l = l.split('#', 1)[0]
+                if l.startswith('SELINUXTYPE='):
+                    _, policy = l.split('=')
+    for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')):
+        os.unlink(sshkey)
+    for sshkey in glob.glob('/etc/ssh/*_key*'):
+        newkey = os.path.join(rootdir, sshkey[1:])
+        shutil.copy2(sshkey, newkey)
+        finfo = os.stat(sshkey)
+        os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID])
+    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/sysconfig/network-scripts/*')):
+        os.unlink(ifcfg)
+    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/NetworkManager/system-connections/*')):
+        os.unlink(ifcfg)
+    for ifcfg in glob.glob('/run/NetworkManager/system-connections/*'):
+        newcfg = ifcfg.split('/')[-1]
+        newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg))
+        shutil.copy2(ifcfg, newcfg)
+    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
+    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    if policy:
+        sys.stdout.write('Applying SELinux labeling...')
+        sys.stdout.flush()
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')])
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')])
+        sys.stdout.write('Done\n')
+        sys.stdout.flush()
+    for metafs in ('proc', 'sys', 'dev'):
+        subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
+    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+        defgrub = defgrubin.read().split('\n')
+    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+        for gline in defgrub:
+            gline = gline.split()
+            newline = []
+            for ent in gline:
+                if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
+                    continue
+                newline.append(ent)
+            defgrubout.write(' '.join(newline) + '\n')
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
+    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    newroot = None
+    with open('/etc/shadow') as shadowin:
+        shents = shadowin.read().split('\n')
+        for shent in shents:
+            shent = shent.split(':')
+            if not shent:
+                continue
+            if shent[0] == 'root' and shent[1] not in ('*', '!!', ''):
+                newroot = shent[1]
+    if newroot:
+        shlines = None
+        with open(os.path.join(rootdir, 'etc/shadow')) as oshadow:
+            shlines = oshadow.read().split('\n')
+        with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow:
+            for line in shlines:
+                if line.startswith('root:'):
+                    line = line.split(':')
+                    line[1] = newroot
+                    line = ':'.join(line)
+                oshadow.write(line + '\n')
+    partnum = None
+    targblock = None
+    for vol in vols:
+        if vol['mount'] == '/boot/efi':
+            targdev = vol['targetdisk']
+            partnum = re.search('(\d+)$', targdev).group(1)
+            targblock = re.search('(.*)\d+$', targdev).group(1)
+    if targblock:
+        shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
+        shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
+        subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+    #other network interfaces
+
+
+def had_swap():
+    with open('/etc/fstab') as tabfile:
+        tabs = tabfile.read().split('\n')
+        for tab in tabs:
+            tab = tab.split()
+            if len(tab) < 3:
+                continue
+            if tab[2] == 'swap':
+                return True
+    return False
+
+def install_to_disk(imgpath):
+    lvmvols = {}
+    deftotsize = 0
+    mintotsize = 0
+    deflvmsize = 0
+    minlvmsize = 0
+    biggestsize = 0
+    biggestfs = None
+    plainvols = {}
+    allvols = []
+    swapsize = 0
+    if had_swap():
+        with open('/proc/meminfo') as meminfo:
+            swapsize = meminfo.read().split('\n')[0]
+        swapsize = int(swapsize.split()[1])
+        if swapsize < 2097152:
+            swapsize = swapsize * 2
+        elif swapsize > 8388608 and swapsize < 67108864:
+            swapsize = swapsize * 0.5
+        elif swapsize >= 67108864:
+            swapsize = 33554432
+        swapsize = int(swapsize * 1024)
+    deftotsize = swapsize
+    mintotsize = swapsize
+    for fs in get_image_metadata(imgpath):
+        allvols.append(fs)
+        deftotsize += fs['initsize']
+        mintotsize += fs['minsize']
+        if fs['initsize'] > biggestsize:
+            biggestfs = fs
+            biggestsize = fs['initsize']
+        if fs['device'].startswith('/dev/mapper'):
+            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
+            deflvmsize += fs['initsize']
+            minlvmsize += fs['minsize']
+        else:
+            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+    with open('/tmp/installdisk') as diskin:
+        instdisk = diskin.read()
+    instdisk = '/dev/' + instdisk
+    parted = PartedRunner(instdisk)
+    dinfo = parted.run('unit s print')
+    dinfo = dinfo.split('\n')
+    sectors = 0
+    sectorsize = 0
+    for inf in dinfo:
+        if inf.startswith('Disk {0}:'.format(instdisk)):
+            _, sectors = inf.split(': ')
+            sectors = int(sectors.replace('s', ''))
+        if inf.startswith('Sector size (logical/physical):'):
+            _, sectorsize = inf.split(':')
+            sectorsize = sectorsize.split('/')[0]
+            sectorsize = sectorsize.replace('B', '')
+            sectorsize = int(sectorsize)
+    # for now, only support resizing/growing the largest partition
+    minexcsize = deftotsize - biggestfs['initsize']
+    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
+    minsectors = mintotsize // sectorsize
+    if sectors < (minsectors + 65536):
+        raise Exception('Disk too small to fit image')
+    biggestsectors = sectors - (minexcsize // sectorsize)
+    biggestsize = sectorsize * biggestsectors
+    parted.run('mklabel gpt')
+    curroffset = 2048
+    for volidx in sorted(plainvols):
+        vol = plainvols[volidx]
+        if vol is not biggestfs:
+            size = vol['initsize'] // sectorsize
+        else:
+            size = biggestsize // sectorsize
+        size += 2047 - (size % 2048)
+        end = curroffset + size
+        if end > sectors:
+            end = sectors
+        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
+        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        curroffset += size + 1
+    if not lvmvols:
+        if swapsize:
+            swapsize = swapsize // sectorsize
+            swapsize += 2047 - (size % 2048)
+            end = curroffset + swapsize
+            if end > sectors:
+                end = sectors
+            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
+            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+    else:
+        parted.run('mkpart lvm {}s 100%'.format(curroffset))
+        lvmpart = instdisk + '{}'.format(volidx + 1)
+        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
+        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
+        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
+        vginfo = vginfo.split('\n')
+        pesize = 0
+        pes = 0
+        for infline in vginfo:
+            infline = infline.split()
+            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
+                pesize = int(infline[2])
+            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
+                pes = int(infline[4])
+        takeaway = swapsize // pesize
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                continue
+            takeaway += vol['initsize'] // pesize
+            takeaway += 1
+        biggestextents = pes - takeaway
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                extents = biggestextents
+            else:
+                extents = vol['initsize'] // pesize
+                extents += 1
+            if vol['mount'] == '/':
+                lvname = 'root'
+            else:
+                lvname = vol['mount'].replace('/', '_')
+            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
+            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
+        if swapsize:
+            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
+            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
+        os.makedirs('/run/imginst/targ')
+        for vol in allvols:
+            with open(vol['targetdisk'], 'wb') as partition:
+                partition.write(b'\x00' * 1 * 1024 * 1024)
+            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
+            source = vol['mount'].replace('/', '_')
+            source = '/run/imginst/sources/' + source
+            blankfsstat = os.statvfs('/run/imginst/targ')
+            blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
+            sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
+            with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier:
+                stillrunning = copier.poll()
+                lastprogress = 0.0
+                while stillrunning is None:
+                    currfsstat = os.statvfs('/run/imginst/targ')
+                    currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize
+                    currused -= blankused
+                    with open('/proc/meminfo') as meminf:
+                        for line in meminf.read().split('\n'):
+                            if line.startswith('Dirty:'):
+                                _, dirty, _ = line.split()
+                                dirty = int(dirty) * 1024
+                    progress = (currused - dirty) / vol['minsize']
+                    if progress < lastprogress:
+                        progress = lastprogress
+                    if progress > 0.99:
+                        progress = 0.99
+                    lastprogress = progress
+                    progress = progress * 100
+                    sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                    sys.stdout.flush()
+                    time.sleep(0.5)
+                    stillrunning = copier.poll()
+                if stillrunning != 0:
+                    raise Exception("Error copying volume")
+                with subprocess.Popen(['sync']) as syncrun:
+                    stillrunning = syncrun.poll()
+                    while stillrunning is None:
+                        with open('/proc/meminfo') as meminf:
+                            for line in meminf.read().split('\n'):
+                                if line.startswith('Dirty:'):
+                                    _, dirty, _ = line.split()
+                                    dirty = int(dirty) * 1024
+                        progress = (vol['minsize'] - dirty) / vol['minsize']
+                        if progress < lastprogress:
+                            progress = lastprogress
+                        if progress > 0.99:
+                            progress = 0.99
+                        lastprogress = progress
+                        progress = progress * 100
+                        sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                        sys.stdout.flush()
+                        time.sleep(0.5)
+                        stillrunning = syncrun.poll()
+                sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+            subprocess.check_call(['umount', '/run/imginst/targ'])
+        for vol in allvols:
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
+        fixup('/run/imginst/targ', allvols)
+
+
+if __name__ == '__main__':
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el7-diskless/profiles/default/scripts/imageboot.sh

@@ -0,0 +1,128 @@
+. /lib/dracut-lib.sh
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
+if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+    mount -t tmpfs untethered /mnt/remoteimg
+    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
+else
+    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    modprobe fuse
+    /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
+fi
+/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
+modprobe loop
+loopdev=$(losetup -f)
+export mountsrc=$loopdev
+losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
+if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
+    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+        sleep 1
+    done
+    cipher=$(head -n 1 /tmp/rootimg.key)
+    key=$(tail -n 1 /tmp/rootimg.key)
+    len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
+    len=$(((len-4096)/512))
+    dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
+    /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
+    mountsrc=/dev/mapper/cryptimg
+fi
+
+if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then
+    mount -o ro $mountsrc /mnt/remote
+elif grep  '^Format: confluent_multisquash' /tmp/rootimg.info; then
+    tail -n +3 /tmp/rootimg.info  | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
+    . /tmp/setupmount.sh
+    cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
+    . /tmp/mountparts.sh
+fi
+
+
+#mount -t tmpfs overlay /mnt/overlay
+modprobe zram
+memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+echo $memtot > /sys/block/zram0/disksize
+mkfs.xfs /dev/zram0 > /dev/null
+mount -o discard /dev/zram0 /mnt/overlay
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /mnt/overlay/upper /mnt/overlay/work
+    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+    done
+fi
+mkdir -p /sysroot/etc/ssh
+mkdir -p /sysroot/etc/confluent
+mkdir -p /sysroot/root/.ssh
+cp /root/.ssh/* /sysroot/root/.ssh
+chmod 700 /sysroot/root/.ssh
+cp /etc/confluent/* /sysroot/etc/confluent/
+cp /etc/ssh/*key* /sysroot/etc/ssh/
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
+done
+
+mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
+if [ ! -z "$autocons" ]; then
+    autocons=${autocons%,*}
+    mkdir -p /run/systemd/generator/getty.target.wants
+    ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
+fi
+while [ ! -e /sysroot/sbin/init ]; do
+    echo "Failed to access root filesystem or it is missing /sbin/init"
+    echo "System should be accessible through ssh at port 2222 with the appropriate key"
+    while [ ! -e /sysroot/sbin/init ]; do
+        sleep 1
+    done
+done
+rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
+rootpassword=${rootpassword#rootpassword: }
+if [ "$rootpassword" = "null" ]; then
+    rootpassword=""
+fi
+
+if [ ! -z "$rootpassword" ]; then
+    sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
+fi
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
+sshconf=/sysroot/etc/ssh/ssh_config
+if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
+cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
+chmod 640 /sysroot/etc/ssh/*_key
+chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
+chroot /sysroot/ update-ca-trust
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
+mkdir -p /sysroot/opt/confluent/bin
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
+chmod +x /sysroot/opt/confluent/bin/onboot.sh
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
+ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
+cp /etc/confluent/functions /sysroot/etc/confluent/functions
+if grep installtodisk /proc/cmdline > /dev/null; then
+    . /etc/confluent/functions
+    run_remote installimage
+    exec reboot -f
+fi
+mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
+ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+exec /opt/confluent/bin/start_root

confluent_osdeploy/el7-diskless/profiles/default/scripts/installimage

@@ -0,0 +1,43 @@
+#!/bin/bash
+. /etc/confluent/functions
+# the image will be used to deploy itself
+# provide both access to image (for parsing metadata)
+# and existing mounts of image (to take advantage of caching)
+mount -o bind /sys /sysroot/sys
+mount -o bind /dev /sysroot/dev
+mount -o bind /proc /sysroot/proc
+mount -o bind /run /sysroot/run
+
+
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /sysroot/run/imginst/sources/_
+    mount -o bind /mnt/remote /sysroot/run/imginst/sources/_
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do
+        srcname=${srcmount#/dev/mapper/mproot}
+        srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g')
+        mkdir -p /sysroot/run/imginst/sources/$srcname
+        mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname
+    done
+fi
+cd /sysroot/run
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+if [ ! -f /sysroot/tmp/installdisk ]; then
+    echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
+    while [ ! -f /sysroot/tmp/installdisk ]; do
+        sleep 1
+    done
+fi
+lvm vgchange -a n
+udevadm control -e
+chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
+chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
+echo "Port 22" >> /etc/ssh/sshd_config
+echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
+echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+
+chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
+chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Confluent onboot hook
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/onboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script is executed on each boot as it is
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-onboot.log
+exec 2>> /var/log/confluent/confluent-onboot.log
+chmod 600 /var/log/confluent/confluent-onboot.log
+tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
+logshowpid=$!
+
+run_remote_python syncfileclient
+run_remote_python confignet
+
+run_remote onboot.custom
+# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
+run_remote_parts onboot.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
+run_remote_config onboot.d
+
+#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+kill $logshowpid

confluent_osdeploy/el7-diskless/profiles/default/scripts/post.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# This script is executed 'chrooted' into a cloned disk target before rebooting
+#
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-post.log
+exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
+tail -f /var/log/confluent/confluent-post.log > /dev/console &
+logshowpid=$!
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+mkdir -p /opt/confluent/bin
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+systemctl enable firstboot
+selinuxpolicy=$(grep ^SELINUXTYPE /etc/selinux/config |awk -F= '{print $2}')
+if [ ! -z "$selinuxpolicy" ]; then
+    setfiles /etc/selinux/${selinuxpolicy}/contexts/files/file_contexts /etc/
+fi
+run_remote_python syncfileclient
+run_remote post.custom
+# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh
+run_remote_parts post.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post.d
+
+curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+
+kill $logshowpid
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1,279 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+try:
+    from importlib.machinery import SourceFileLoader
+    def load_source(mod, path):
+        return SourceFileLoader(mod, path).load_module()
+except ImportError:
+    from imp import load_source
+
+try:
+    apiclient = load_source('apiclient', '/opt/confluent/bin/apiclient')
+except IOError:
+    apiclient = load_source('apiclient', '/etc/confluent/apiclient')
+
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/el7-diskless/profiles/default/syncfiles

@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -18,4 +18,5 @@ mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
 done
-cp /opt/confluent/bin/apiclient /sysroot/etc/confluent
+mkdir -p /sysroot/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin

confluent_osdeploy/el7/profiles/default/syncfiles

@@ -1,5 +1,13 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
 # This file lists files to synchronize or merge to the deployed systems from the deployment server
-# To specify taking /some/path/hosts on the deploymennt server and duplicating it to /etc/hosts:
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
 # /some/path/hosts -> /etc/hosts
 
 # If wanting to simply use the same path for source and destinaiton, the -> may be skipped:

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -8,6 +8,7 @@ get_remote_apikey() {
         confluent_apikey=$(cat /etc/confluent/confluent.apikey)
         if [ -z "$confluent_apikey" ]; then
             echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
+            if [ ! -z "$autoconsdev" ]; then echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..." > $autoconsdev; fi
             sleep 10
         elif [ -c /dev/tpmrm0 ]; then
             tmpdir=$(mktemp -d)
@@ -38,6 +39,7 @@ echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
 
 if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)
+    autoconsdev=${autocons%,*}
     autocons=${autocons##*/}
     echo "Automatic console configured for $autocons"
 fi
@@ -129,12 +131,18 @@ while [ $ready = "0" ]; do
         fi
         confluent_mgr=${confluent_mgr#[}
         confluent_mgr=${confluent_mgr%]}
+    elif grep 'SSL' $tmperr > /dev/null; then
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    	echo 'Failure establishing TLS conneection to '$confluent_mgr' (try `osdeploy initialize -t` on the deployment server)'
+	if [ ! -z "$autoconsdev" ]; then echo 'Failure establishing TLS conneection to '$confluent_mgr' (try `osdeploy initialize -t` on the deployment server)' > $autoconsdev; fi
+    	sleep 10
     else
         ready=1
     fi
     rm $tmperr
 done
-if [ ! -z "$autocons" ] && grep textconsole: true /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
+if [ ! -z "$autocons" ] && grep "textconsole: true" /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
 if [ -c /dev/tpmrm0 ]; then
     tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
 fi

confluent_osdeploy/el8-diskless/profiles/default/syncfiles

@@ -1,5 +1,13 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
 # This file lists files to synchronize or merge to the deployed systems from the deployment server
-# To specify taking /some/path/hosts on the deploymennt server and duplicating it to /etc/hosts:
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
 # /some/path/hosts -> /etc/hosts
 
 # If wanting to simply use the same path for source and destinaiton, the -> may be skipped:

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 echo -n "" >> /tmp/net.ifaces
+echo -n "" > /tmp/01-autocons.devnode
 cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
 if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -19,4 +19,5 @@ mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
 done
-cp /opt/confluent/bin/apiclient /sysroot/etc/confluent
+mkdir -p /sysroot/opt/confuent/bin
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh

@@ -42,6 +42,7 @@ cd /sys/class/net
 while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
     TRIES=$((TRIES + 1))
     for currif in *; do
+        echo 0 > /proc/sys/net/ipv6/conf/${currif}/autoconf
         ip link set $currif up
     done
     /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
@@ -56,7 +57,14 @@ nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 #TODO: blkid --label <whatever> to find mounted api
 
 cat /tls/*.pem > /etc/confluent/ca.pem
-confluentpython /opt/confluent/bin/apiclient /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+autocons=$(cat /tmp/01-autocons.devnode)
+errout=""
+if [ ! -z "$autocons" ]; then
+    errout="-e $autocons"
+fi
+while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg; do
+	sleep 10
+done
 ifidx=$(cat /tmp/confluent.ifidx)
 ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
 ifname=${ifname%:}
@@ -92,6 +100,7 @@ if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; the
 	if [ ! -z "$autocons" ]; then
 	    echo Auto-configuring installed system to use text console
 	    echo Auto-configuring installed system to use text console > $autocons
+            /opt/confluent/bin/autocons -c > /dev/null
 	    cp /tmp/01-autocons.conf /etc/cmdline.d/
 	else
 	    echo "Unable to automatically detect requested text console"
@@ -136,10 +145,14 @@ if [ "$v6cfg" = "static" ]; then
     echo ip=$v6addr::$v6gw:$v6nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
 fi
 nameserversec=0
+v4dns=0
+v6dns=0
 while read -r entry; do
     if [ $nameserversec = 1 ]; then
         if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
             echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            [[ "$entry" == *:* ]] && v6dns=1
+            [[ "$entry" == *.* ]] && v4dns=1
             continue
         fi
     fi
@@ -152,6 +165,24 @@ done < /etc/confluent/confluent.deploycfg
 if [ -e /lib/nm-lib.sh ]; then
     . /lib/nm-lib.sh
     nm_generate_connections
+    if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+        grep -v ^dns-search= /run/NetworkManager/system-connections/$ifname.nmconnection > /run/NetworkManager/system-connections/$ifname.nmconnection.new
+        mv /run/NetworkManager/system-connections/$ifname.nmconnection.new /run/NetworkManager/system-connections/$ifname.nmconnection
+        if [ "$v4dns" = 1 ]; then
+            awk '/^\[ipv4\]/ {print;print "dns-search='"$dnsdomain"'";next}1' /run/NetworkManager/system-connections/$ifname.nmconnection > /run/NetworkManager/system-connections/$ifname.nmconnection.new
+        else
+            awk '/^\[ipv4\]/ {print;print "dns-search=";next}1' /run/NetworkManager/system-connections/$ifname.nmconnection > /run/NetworkManager/system-connections/$ifname.nmconnection.new
+        fi
+        mv /run/NetworkManager/system-connections/$ifname.nmconnection.new /run/NetworkManager/system-connections/$ifname.nmconnection
+        if [ "$v6dns" = 1 ]; then
+            awk '/^\[ipv6\]/ {print;print "dns-search='"$dnsdomain"'";next}1' /run/NetworkManager/system-connections/$ifname.nmconnection > /run/NetworkManager/system-connections/$ifname.nmconnection.new
+        else
+            awk '/^\[ipv6\]/ {print;print "dns-search=";next}1' /run/NetworkManager/system-connections/$ifname.nmconnection > /run/NetworkManager/system-connections/$ifname.nmconnection.new
+        fi
+        mv /run/NetworkManager/system-connections/$ifname.nmconnection.new /run/NetworkManager/system-connections/$ifname.nmconnection
+        chmod 600 /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+
     if [[ "$ifname" == ib* ]]; then
         sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
         if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then

confluent_osdeploy/el8/profiles/default/kickstart

@@ -94,6 +94,7 @@ chmod +x /mnt/sysimage/opt/confluent/bin/firstboot.sh
 
 %post
 cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+cp /etc/confluent/tls/*.pem /etc/pki/ca-trust/source/anchors
 systemctl enable firstboot
 chgrp ssh_keys /etc/ssh/ssh*key
 restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /opt/confluent/bin/firstboot.sh

confluent_osdeploy/el8/profiles/default/syncfiles

@@ -1,5 +1,13 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
 # This file lists files to synchronize or merge to the deployed systems from the deployment server
-# To specify taking /some/path/hosts on the deploymennt server and duplicating it to /etc/hosts:
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
 # /some/path/hosts -> /etc/hosts
 
 # If wanting to simply use the same path for source and destinaiton, the -> may be skipped:

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -76,8 +76,10 @@ for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
 done
 cd - > /dev/null
 rm -rf $tpmdir
-/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
-if [ ! -z "$autocons" ] && grep textconsole: true /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
+while ! /usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg; do
+	sleep 10
+done
+if [ ! -z "$autocons" ] && grep "textconsole: true" /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
 umask $oldumask
 new_apikey=$(cat /etc/confluent/confluent.apikey)
 if [ "$new_apikey" != "$confluent_apikey" ]; then
@@ -153,8 +155,8 @@ done
 /usr/sbin/sshd
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 export confluent_profile
-/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /tmp/functions
-. /tmp/functions
+/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
 set_confluent_vars
 export confluent_mgr
 echo "Running https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh"

confluent_osdeploy/genesis/profiles/default/scripts/onboot.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-. /tmp/functions
+. /etc/confluent/functions
 # This runs whenever this genesis profile boots for customization
 # purposes
 

confluent_osdeploy/suse15/profiles/hpc/scripts/prechroot.sh

@@ -18,7 +18,8 @@ cp -a /tls/* /mnt/var/lib/ca-certificates/openssl
 cp -a /tls/* /mnt/var/lib/ca-certificates/pem
 cp -a /tls/*.pem /mnt/etc/pki/trust/anchors
 cat /tls/*.pem > /mnt/etc/confluent/ca.pem
-cp /opt/confluent/bin/apiclient /mnt/etc/confluent
+mkdir -p /mnt/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /opt/confluent/bin/
 
 run_remote setupssh.sh
 

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -49,8 +49,9 @@ fi
 mkdir -p /opt/confluent/bin
 mkdir -p /etc/confluent
 cp -a /target/etc/confluent/* /etc/confluent
+mkdir -p /target/opt/confluent/bin
 cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin/
-cp /custom-installation/confluent/bin/apiclient /target/etc/confluent/
+cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin
 
 mount -o bind /dev /target/dev
 mount -o bind /proc /target/proc

confluent_osdeploy/utils/autocons.c

@@ -62,6 +62,10 @@ int main(int argc, char* argv[]) {
         exit(0);
     }
     ttyf = open(buff, O_RDWR | O_NOCTTY);
+    if (ttyf < 0) {
+        fprintf(stderr, "Unable to open tty\n");
+        exit(1);
+    }
     if (currspeed == SPEED9600) {
         cspeed = B9600;
         strcpy(offset, ",9600");
@@ -87,7 +91,9 @@ int main(int argc, char* argv[]) {
     cfmakeraw(&tty2);
     tcsetattr(ttyf, TCSANOW, &tty2);
     flags = fcntl(ttyf, F_GETFL, 0);
-    fcntl(ttyf, F_SETFL, flags | O_NONBLOCK);
+    if (fcntl(ttyf, F_SETFL, flags | O_NONBLOCK) < 0) {
+        fprintf(stderr, "Failed setting flags on tty\n");
+    }
     while (read(ttyf, buff, 64) > 0) {
         // Drain any pending reads
     }
@@ -124,7 +130,7 @@ int main(int argc, char* argv[]) {
     if (ws.ws_row < 24) { ws.ws_col = 24; }
     ioctl(ttyf, TIOCSWINSZ, &ws);
     tcsetattr(ttyf, TCSANOW, &tty);
-    if (argc > 1 && (strcmp(argv[1], '-c') == 0)) {
+    if (argc > 1 && (strcmp(argv[1], "-c") == 0)) {
         ioctl(ttyf, TIOCCONS, 0);
     }
 }

confluent_osdeploy/utils/clortho.c

@@ -23,11 +23,16 @@ unsigned char* genpasswd(int len) {
         int urandom, ret;
         passwd = calloc(len + 1, sizeof(char));
         urandom = open("/dev/urandom", O_RDONLY);
+        if (urandom < 0) {
+            fprintf(stderr, "Failed reading /dev/urandom\n");
+            exit(1);
+        }
         ret = read(urandom, passwd, len);
         close(urandom);
         for (urandom = 0; urandom < len; urandom++) {
                 passwd[urandom] = cryptalpha[passwd[urandom] >> 2];
         }
+        passwd[len] = 0;  // Should be redundant with calloc, but be explicit
         return passwd;
 
 }
@@ -80,7 +85,9 @@ int main(int argc, char* argv[]) {
         for (curr = addrs; curr != NULL; curr = curr->ai_next) {
                 sock = socket(curr->ai_family, curr->ai_socktype, curr->ai_protocol);
                 if (sock < 0) continue;
-                setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
+                if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int)) < 0) {
+                    fprintf(stderr, "Failed setting reusaddr\n");
+                }
                 if (curr->ai_family == AF_INET) {
                     if (bind(sock, (struct sockaddr*)&net4bind, sizeof(struct sockaddr_in)) < 0) {
                         fprintf(stderr, "Unable to bind port 302\n");
@@ -100,7 +107,9 @@ int main(int argc, char* argv[]) {
                 fprintf(stderr, "Unable to reach %s\n", argv[2]);
                 exit(1);
         }
-        setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
+        if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) < 0) {
+            fprintf(stderr, "Unable to set timeout\n");
+        }
         freeaddrinfo(addrs);
         ret = read(sock, buffer, 8);
         if (memcmp(buffer, "\xc2\xd1-\xa8\x80\xd8j\xba", 8) != 0) {
@@ -124,7 +133,7 @@ int main(int argc, char* argv[]) {
             memset(buffer, 0, MAXPACKET);
 	    if (currlen > 1000) {
                 fprintf(stderr, "Received oversized message\n");
-		exit(1);
+                exit(1);
             }
             if (currlen) {
                 ret = read(sock, buffer, currlen);  // Max is 1000, well under MAX_PACKET
@@ -137,9 +146,9 @@ int main(int argc, char* argv[]) {
                 dprintf(sock, "\x04%c%s", slen, cryptedpass);
                 ret = write(sock, "\x00\x00", 2);
             } else if (currtype == 128) {
-		printf("SEALED:%s", buffer);
-		printf("\n");
-		exit(0);
+                printf("SEALED:%s", buffer);
+                printf("\n");
+                exit(0);
             } else if (currtype == 5) {
                 printf("%s", passwd);
                 printf("\n");

confluent_osdeploy/utils/confluent_imginfo.c

@@ -4,42 +4,90 @@
 #include <string.h>
 
 int read_part(FILE* img, long int imgsize) {
-    char mountpath[65537];
-    char devpath[65537];
-    char fstype[65537];
+    char * mountpath;
+    char * devpath;
+    char * fstype;
     uint16_t shortlength;
     uint32_t length;
     uint64_t longlength;
-    fread(&shortlength, 2, 1, img);
+    if (fread(&shortlength, 2, 1, img) < 2) {
+        fprintf(stderr, "Error reading section\n");
+        exit(1);
+    }
     shortlength = be16toh(shortlength);
-    fread(mountpath, 1, shortlength, img);
+    mountpath = (char*)malloc(shortlength + 1);
+    if (fread(mountpath, 1, shortlength, img) < shortlength) {
+        fprintf(stderr, "Failure reading segment\n");
+        exit(1);
+    }
     mountpath[shortlength] = 0;
-    fread(&length, 4, 1, img);
+    if (fread(&length, 4, 1, img) < 4) {
+        fprintf(stderr, "Failure reading segment\n");
+        exit(1);
+    }
     length = be32toh(length);
-    fseek(img, length, SEEK_CUR); // skip json section that we don't support
-    fread(&longlength, 8, 1, img); // minimum size in bytes
+    if (fseek(img, length, SEEK_CUR) != 0) { // skip json section that we don't support
+        fprintf(stderr, "Error skipping json segment");
+        exit(1);
+    }
+    if (fread(&longlength, 8, 1, img) < 8) { // minimum size in bytes
+        fprintf(stderr, "Failure reading segment\n");
+        exit(1);
+    }
     longlength = be64toh(longlength);
     printf("%ld\t", longlength);
-    fread(&longlength, 8, 1, img); // default size in bytes
+    if (fread(&longlength, 8, 1, img) < 8) { // default size in bytes
+        fprintf(stderr, "Error reading segment\n");
+        exit(1);
+    }
     longlength = be64toh(longlength);
     printf("%ld\t", longlength);
-    fread(&shortlength, 2, 1, img); // length of filesystem type
+    if (fread(&shortlength, 2, 1, img) < 2) { // length of filesystem type
+        fprintf(stderr, "Error reading segment\n");
+        exit(1);
+    }
     shortlength = be16toh(shortlength);
-    fread(fstype, 1, shortlength, img);
+    fstype = (char*)malloc(shortlength + 1);
+    if (fread(fstype, 1, shortlength, img) < shortlength) {
+        fprintf(stderr, "Error reading segment\n");
+        exit(1);
+    }
     fstype[shortlength] = 0;
-    fread(&shortlength, 2, 1, img); // length of DEVICE
+    if (fread(&shortlength, 2, 1, img) < 2) { // length of DEVICE
+        fprintf(stderr, "Error reading segment\n");
+        exit(1);
+    }
     shortlength = be16toh(shortlength);
-    fread(devpath, 1, shortlength, img);
+    devpath = (char*)malloc(shortlength + 1);
+    if (fread(devpath, 1, shortlength, img) < shortlength) {
+        fprintf(stderr, "Error reading segment\n");
+        exit(1);
+    }
     devpath[shortlength] = 0;
-    fread(&shortlength, 2, 1, img);
+    if (fread(&shortlength, 2, 1, img) < 2) {
+        fprintf(stderr, "Error reading segment\n");
+        exit(1);
+    }
     shortlength = be16toh(shortlength);
-    fseek(img, shortlength, SEEK_CUR); // Skip the padding
-    fread(&longlength, 8, 1, img);
+    if (fseek(img, shortlength, SEEK_CUR) != 0) { // Skip the padding
+        fprintf(stderr, "Failure skipping padding\n");
+        exit(1);
+    }
+    if (fread(&longlength, 8, 1, img) < 8) {
+        fprintf(stderr, "Error reading section\n");
+        exit(1);
+    }
     longlength = be64toh(longlength);
     printf("%ld\t", ftell(img) / 512);
     printf("%ld\t", longlength / 512);
     printf("%s\t%s\t%s\n", fstype, devpath, mountpath);
-    fseek(img, longlength, SEEK_CUR);
+    free(mountpath);
+    free(devpath);
+    free(fstype);
+    if (fseek(img, longlength, SEEK_CUR) != 0) {
+        fprintf(stderr, "Error restoring seek\n");
+        exit(1);
+    }
     return (ftell(img) < imgsize);
 }
 
@@ -62,15 +110,30 @@ int main(int argc, char* argv[]) {
     }
     if (memcmp(buffer, "\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1", 16) == 0) {
         printf("Format: confluent_multisquash\nminsize\tdefsize\toffset\tsize\tfstype\torigdev\tmount\n");
-        fread(buffer, 1, 1, img);
-        fseek(img, buffer[0], SEEK_CUR);
+        if (fread(buffer, 1, 1, img) < 1) {
+            fprintf(stderr, "Error reading image\n");
+            exit(1);
+        }
+        if (fseek(img, buffer[0], SEEK_CUR) != 0) {
+            fprintf(stderr, "Error seeking in image\n");
+            exit(1);
+        }
         while (read_part(img, imgsize));
         exit(0);
     }
     if (memcmp(buffer, "\xaa\xd5\x0f\x7e\x5d\xfb\x4b\x7c\xa1\x2a\xf4\x0b\x6d\x94\xf7\xfc", 16) == 0) {
-        fread(buffer, 1, 1, img);
-        fseek(img, buffer[0], SEEK_CUR);
-        fread(buffer, 1, 1, img);
+        if (fread(buffer, 1, 1, img) < 1) {
+            fprintf(stderr, "Error reading image\n");
+            exit(1);
+        }
+        if (fseek(img, buffer[0], SEEK_CUR) != 0) {
+            fprintf(stderr, "Error reading image\n");
+            exit(1);
+        }
+        if (fread(buffer, 1, 1, img) < 1) {
+            fprintf(stderr, "Error reading image\n");
+            exit(1);
+        }
         if (buffer[0] == 0) {
             printf("Format: confluent_crypted\n");
             exit(0);

confluent_osdeploy/utils/copernicus.c

@@ -13,11 +13,21 @@
 #include <sys/time.h>
 #include <net/if.h>
 
+typedef struct sockaddr_llib {
+    unsigned short int sll_family;
+    unsigned short int sll_protocol;
+    int sll_ifindex;
+    unsigned short int sll_hatype;
+    unsigned char sll_pkttype;
+    unsigned char sll_halen;
+    unsigned char sll_addr[20];
+} sockaddr_llib;
+
 int add_uuid(char* destination, int maxsize) {
     int uuidf;
     int uuidsize;
     uuidf = open("/sys/devices/virtual/dmi/id/product_uuid", O_RDONLY);
-    if (uuidf < 1) { return 0; }
+    if (uuidf < 0) { return 0; }
     strncpy(destination, "/uuid=", maxsize);
     uuidsize = read(uuidf, destination + 6, maxsize - 6);
     close(uuidf);
@@ -27,9 +37,23 @@ int add_uuid(char* destination, int maxsize) {
     return  uuidsize + 6;
 }
 
+int add_confluent_uuid(char* destination, int maxsize) {
+    int uuidf;
+    int uuidsize;
+    uuidf = open("/confluent_uuid", O_RDONLY);
+    if (uuidf < 0) { return 0; }
+    strncpy(destination, "/confluentuuid=", maxsize);
+    uuidsize = read(uuidf, destination + 15, maxsize - 15);
+    close(uuidf);
+    if (destination[uuidsize + 14] == '\n') {
+        destination[uuidsize + 14] = 0;
+    }
+    return uuidsize + 15;
+}
+
 void add_macs(char* destination, int maxsize) {
     struct ifaddrs *ifc, *ifa;
-    struct sockaddr_ll *lla;
+    struct sockaddr_llib *lla;
     int offset;
     char macaddr[32];
 
@@ -38,7 +62,7 @@ void add_macs(char* destination, int maxsize) {
     for (ifc = ifa; ifc != NULL; ifc = ifc->ifa_next) {
         if (ifc->ifa_addr->sa_family != AF_PACKET)
             continue;
-        lla = (struct sockaddr_ll *)ifc->ifa_addr;
+        lla = (struct sockaddr_llib *)ifc->ifa_addr;
         if (lla->sll_hatype == ARPHRD_INFINIBAND) {
             snprintf(macaddr, 32, "/mac=%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
                 lla->sll_addr[12], lla->sll_addr[13], lla->sll_addr[14],
@@ -105,18 +129,38 @@ int main(int argc, char* argv[]) {
     inet_pton(AF_INET, "239.255.255.250", &dst4.sin_addr);
     strncpy(msg,  "M-SEARCH * HTTP/1.1\r\nST: urn:xcat.org:service:confluent:", 1024);
     offset = strnlen(msg, 1024);
+    add_confluent_uuid(msg + offset, 1024 - offset);
+    offset = strnlen(msg, 1024);
     add_uuid(msg + offset, 1024 - offset);
     offset = strnlen(msg, 1024);
     add_macs(msg + offset, 1024 - offset);
     offset = strnlen(msg, 1024);
     ns = socket(AF_INET6, SOCK_DGRAM, 0);
     n4 = socket(AF_INET, SOCK_DGRAM, 0);
+    if (ns < 0) {
+        fprintf(stderr, "Error opening IPv6 socket\n");
+        exit(1);
+    }
+    if (n4 < 0) {
+        fprintf(stderr, "Error opening IPv4 socket\n");
+        exit(1);
+    }
     ifidx = 1; /* reuse ifidx because it's an unused int here */
-    setsockopt(n4, SOL_SOCKET, SO_BROADCAST, &ifidx, sizeof(ifidx));
-    setsockopt(ns, IPPROTO_IPV6, IPV6_V6ONLY, &ifidx, sizeof(ifidx));
+    if (setsockopt(n4, SOL_SOCKET, SO_BROADCAST, &ifidx, sizeof(ifidx)) != 0) {
+        fprintf(stderr, "Unable to set broadcast on socket\n");
+    }
+    if (setsockopt(ns, IPPROTO_IPV6, IPV6_V6ONLY, &ifidx, sizeof(ifidx)) != 0) {
+        fprintf(stderr, "Unable to limit socket to IPv6 only\n");
+    }
     /* For now, bind to 190 to prove we are a privileged process */
-    bind(n4, (const struct sockaddr *)&addr4, sizeof(addr4));
-    bind(ns, (const struct sockaddr *)&addr, sizeof(addr));
+    if (bind(n4, (const struct sockaddr *)&addr4, sizeof(addr4)) < 0) {
+        fprintf(stderr, "Eror binding privilged port!\n");
+        exit(1);
+    }
+    if (bind(ns, (const struct sockaddr *)&addr, sizeof(addr)) < 0) {
+        fprintf(stderr, "Error binding ipv6 privileged port!\n");
+        exit(1);
+    }
     getifaddrs(&ifa);
     for (ifc = ifa; ifc != NULL; ifc = ifc->ifa_next) {
         if (!ifc->ifa_addr) continue;
@@ -127,15 +171,23 @@ int main(int argc, char* argv[]) {
             if (in6->sin6_scope_id == 0)
                 continue;
             ifidx = in6->sin6_scope_id;
-            setsockopt(ns, IPPROTO_IPV6, IPV6_MULTICAST_IF, &ifidx, sizeof(ifidx));
-            sendto(ns, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)&dst, sizeof(dst));
+            if (setsockopt(ns, IPPROTO_IPV6, IPV6_MULTICAST_IF, &ifidx, sizeof(ifidx)) != 0)
+                continue;
+            if (sendto(ns, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)&dst, sizeof(dst)) < 0) {
+                continue;
+            }
         } else if (ifc->ifa_addr->sa_family == AF_INET) {
             in = (struct sockaddr_in *)ifc->ifa_addr;
             bin = (struct sockaddr_in *)ifc->ifa_ifu.ifu_broadaddr;
             bin->sin_port = htons(1900);
-            setsockopt(n4, IPPROTO_IP, IP_MULTICAST_IF, &in->sin_addr, sizeof(in->sin_addr));
-            sendto(n4, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)&dst4, sizeof(dst4));
-            sendto(n4, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)bin, sizeof(*bin));
+            if (setsockopt(n4, IPPROTO_IP, IP_MULTICAST_IF, &in->sin_addr, sizeof(in->sin_addr)) != 0)
+                continue;
+            if (sendto(n4, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)&dst4, sizeof(dst4)) < 0) {
+                // ignore failure to send, we are trying to be opportunistic
+            }
+            if (sendto(n4, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)bin, sizeof(*bin)) < 0) {
+                continue;
+            }
         }
     }
     FD_ZERO(&rfds);

confluent_osdeploy/utils/urlmount.c

@@ -39,9 +39,9 @@ typedef struct downloadbuffer {
 
 #define MAX_FILE_LEN 1024
 #define MAX_URL_PATHS 512
-static char filename[MAX_FILE_LEN];
+static char filename[MAX_FILE_LEN + 1];
 static int urlidx, newidx;
-static char* urls[MAX_URL_PATHS];
+static char* urls[MAX_URL_PATHS + 1];
 
 
 void *http_rechecker(void *argp) {
@@ -50,21 +50,36 @@ void *http_rechecker(void *argp) {
     tmpidx = open("/dev/urandom", O_RDONLY);
     if (tmpidx <= 0 || read(tmpidx, (char*)&tmpval, 4) < 0)
         tmpval = time(NULL);
-    if (tmpidx > 0)
+    if (tmpidx >= 0)
         close(tmpidx);
     srand(tmpval);
     checkurl = curl_easy_init();
-    curl_easy_setopt(checkurl, CURLOPT_ERRORBUFFER, curlerror);
+    if (curl_easy_setopt(checkurl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Error buffer\n");
+        exit(1);
+    }
     //We want to consider error conditions fatal, rather than
     //passing error text as data
-    curl_easy_setopt(checkurl, CURLOPT_FAILONERROR, 1L);
-    curl_easy_setopt(checkurl, CURLOPT_TIMEOUT, 10L);
-    curl_easy_setopt(checkurl, CURLOPT_NOBODY, 1);
+    if (curl_easy_setopt(checkurl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Error setting timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Error setting nobody\n");
+        exit(1);
+    }
     while (1) {
-        sleep(25 + rand() % 10);  // Spread out retries across systems
+        sleep(25 + tmpval % 10);  // Spread out retries across systems
         tmpidx = 0;
         while (tmpidx < urlidx && tmpidx < newidx && urls[tmpidx] != NULL) {
-            curl_easy_setopt(checkurl, CURLOPT_URL, urls[tmpidx]);
+            if (curl_easy_setopt(checkurl, CURLOPT_URL, urls[tmpidx]) != CURLE_OK) {
+                tmpidx++;
+                continue;
+            }
             if (curl_easy_perform(checkurl) == CURLE_OK)
                 newidx = tmpidx;
             else
@@ -120,8 +135,14 @@ static int http_read(const char *path, char *buf, size_t size, off_t offset,
     if (offset >= filesize) return 0;
     if (offset + size - 1 >= filesize) size = filesize - offset - 1;
     snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
-    curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer);
-    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dlbuf);
+    if (curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer) != CURLE_OK) {
+        fprintf(stderr, "Error setting range\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&dlbuf) != CURLE_OK) {
+        fprintf(stderr, "Error setting writedata\n");
+        exit(1);
+    }
     if (newidx < MAX_URL_PATHS) {
         reconnecting = 1;
         urlidx = newidx;
@@ -129,7 +150,8 @@ static int http_read(const char *path, char *buf, size_t size, off_t offset,
         fd = fopen("/dev/kmsg", "w+");
         fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
         fclose(fd);
-        curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
+        // if fail, carry on and take the error in curl_easy_perform instead
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
     }
     while (curl_easy_perform(curl) != CURLE_OK) {
         reconnecting = 1;
@@ -138,6 +160,8 @@ static int http_read(const char *path, char *buf, size_t size, off_t offset,
         fprintf(fd, "<4>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror);
         fclose(fd);
         urlidx++;
+        if (urlidx > MAX_URL_PATHS)
+            urlidx = 0;
         if (urls[urlidx] == NULL)
             urlidx = 0;
         if (urlidx == startidx) {
@@ -149,7 +173,12 @@ static int http_read(const char *path, char *buf, size_t size, off_t offset,
         fd = fopen("/dev/kmsg", "w+");
         fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
         fclose(fd);
-        curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
+        if (urlidx > MAX_URL_PATHS) {
+            fprintf(stderr, "Maximum url path exceeded\n");
+            exit(1);
+        }
+        // ignore, let the curl_easy_perform get the error
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
     }
     if (reconnecting) {
         fd = fopen("/dev/kmsg", "w+");
@@ -180,13 +209,28 @@ static void* http_init(struct fuse_conn_info *conn) {
     curl_global_init(CURL_GLOBAL_DEFAULT);
     pthread_create(&tid, NULL, http_rechecker, NULL);
     curl = curl_easy_init();
-    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl error buffor\n");
+        exit(1);
+    }
     //We want to consider error conditions fatal, rather than
     //passing error text as data
-    curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
-    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
-    curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer);
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl failonerror\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing url\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl fill buffer\n");
+        exit(1);
+    }
     return NULL;
 }
 
@@ -225,11 +269,20 @@ int main(int argc, char* argv[]) {
     newidx = MAX_URL_PATHS;
     curl_global_init(CURL_GLOBAL_DEFAULT);
     curl = curl_easy_init();
-    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror);
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Unable to set error buffer\n");
+        exit(1);
+    }
     //We want to consider error conditions fatal, rather than
     //passing error text as data
-    curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
-    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L);
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Unable to set fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L) != CURLE_OK) {
+        fprintf(stderr, "Unable to setup curl timeout\n");
+        exit(1);
+    }
     memset(filename, 0, MAX_FILE_LEN);
     for (i=0; i < argc; i++) {
         if (strstr(argv[i], ":") > 0) {
@@ -252,8 +305,12 @@ int main(int argc, char* argv[]) {
     }
     j = urlidx;
     printf("Connecting to %s\n", urls[urlidx]);
-    curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
-    curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Unable to set url\n");
+    }
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Failure setting no body\n");
+    }
     while (curl_easy_perform(curl) != CURLE_OK) {
         fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror);
         if (urls[urlidx] == NULL)
@@ -263,12 +320,20 @@ int main(int argc, char* argv[]) {
             exit(1);
         }
         printf("Connecting to %s\n", urls[urlidx]);
-        curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]);
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+            fprintf(stderr, "Unable to set url\n");
+        }
     }
     printf("Successfully connected to %s\n", urls[urlidx]);
-    curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize);
+    if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize) != CURLE_OK) {
+        fprintf(stderr, "Failed getting content length\n");
+        exit(1);
+    }
     filesize = round(fsize);
-    curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 0) != CURLE_OK) {
+        fprintf(stderr, "Failed setting nobody\n");
+        exit(1);
+    }
     if (filesize < 1) {
         fprintf(stderr, "Unable to reach designated URL\n");
         exit(1);

confluent_server/bin/osdeploy

@@ -121,6 +121,9 @@ def init_confluent_myname():
         return
     mynamedone = True
     hasconfluentuser = None
+    neededuid = os.stat('/etc/confluent').st_uid
+    if neededuid == 0:
+        return
     try:
         hasconfluentuser = pwd.getpwnam('confluent')
     except KeyError:
@@ -247,6 +250,17 @@ def initialize(cmdset):
                 'subject.')
     else:
         didsomething = False
+    if not os.path.exists('/etc/confluent'):
+        sys.stderr.write('Start confluent service prior to initializng OS deployment\n')
+        sys.exit(1)
+    if cmdset.t or cmdset.s or cmdset.a or cmdset.u:
+        neededuid = os.stat('/etc/confluent').st_uid
+        try:
+            os.makedirs('/var/lib/confluent')
+            os.chown('/var/lib/confluent', neededuid, -1)
+        except OSError as e:
+            if e.errno != 17:
+                raise
     if cmdset.u:
         didsomething = True
         if not glob.glob('/root/.ssh/*.pub'):
@@ -283,9 +297,29 @@ def initialize(cmdset):
     if cmdset.l:
         local_node_trust_setup()
     if cmdset.k:
-        with open('/etc/ssh/ssh_known_hosts', 'a+b') as skh:
+        cas = set([])
+        cakeys = set([])
+        try:
+            with open('/etc/ssh/ssh_known_hosts', 'rb') as skh:
+                for line in skh.read().split(b'\n'):
+                    try:
+                        cakey = line.split()[3]
+                        cakeys.add(cakey)
+                    except IndexError:
+                        pass
+                    if line:
+                        cas.add(line)
+        except IOError:
+            pass
+        with open('/etc/ssh/ssh_known_hosts', 'wb') as skh:
+            for ca in cas:
+                skh.write(ca)
+                skh.write(b'\n')
             for cafile in glob.glob('/var/lib/confluent/public/site/ssh/*.ca'):
                 cacert = open(cafile, 'rb').read()
+                cakey = cacert.split()[1]
+                if cakey in cakeys:
+                    continue
                 cacert = b'@cert-authority * ' + cacert
                 skh.write(cacert)
     if cmdset.g:
@@ -307,6 +341,16 @@ def initialize(cmdset):
     opath = os.getcwd()
     os.chdir('/var/lib/confluent/public/site')
     totar = []
+    if not os.path.exists('confluent_uuid'):
+        c = client.Command()
+        for rsp in c.read('/uuid'):
+            uuid = rsp.get('uuid', {}).get('value', None)
+            if uuid:
+                with open('confluent_uuid', 'w') as uuidout:
+                    uuidout.write(uuid)
+                    uuidout.write('\n')
+    totar.append('confluent_uuid')
+    topack.append('confluent_uuid')
     if os.path.exists('ssh'):
         totar.append('ssh')
         topack.append('ssh/')
@@ -409,6 +453,11 @@ def osimport(imagefile):
                         sys.stdout.write('\n')
                         for profile in rsp['profiles']:
                             print('Deployment profile created: {0}'.format(profile))
+                    if rsp['phase'] == 'error':
+                        sys.stderr.write('{0}\n'.format(rsp['error']))
+                        if 'Permission denied' in rsp['error']:
+                            sys.stderr.write('Confluent service unable to write to destination, check that confluent user has access to target\n')
+                        return
                     sys.stdout.flush()
                 else:
                     print(repr(rsp))

confluent_server/confluent/certutil.py

@@ -15,6 +15,15 @@ def get_openssl_conf_location():
     else:
         raise Exception("Cannot find openssl config file")
 
+def normalize_uid():
+    curruid = os.geteuid()
+    neededuid = os.stat('/etc/confluent').st_uid
+    if curruid != neededuid:
+        os.seteuid(neededuid)
+    if os.geteuid() != neededuid:
+        raise Exception('Need to run as root or owner of /etc/confluent')
+    return curruid
+
 def get_ip_addresses():
     lines, _ = util.run(['ip', 'addr'])
     if not isinstance(lines, str):
@@ -76,7 +85,8 @@ def assure_tls_ca():
             if e.errno != 17:
                 raise
         sslcfg = get_openssl_conf_location()
-        tmpconfig = tempfile.mktemp()
+        tmphdl, tmpconfig = tempfile.mkstemp()
+        os.close(tmphdl)
         shutil.copy2(sslcfg, tmpconfig)
         subprocess.check_call(
             ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
@@ -96,11 +106,14 @@ def assure_tls_ca():
             os.remove(tmpconfig)
     fname = '/var/lib/confluent/public/site/tls/{0}.pem'.format(
         collective.get_myname())
+    ouid = normalize_uid()
     try:
         os.makedirs(os.path.dirname(fname))
     except OSError as e:
         if e.errno != 17:
             raise
+    finally:
+        os.seteuid(ouid)
     shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
     hv, _ = util.run(
         ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
@@ -139,9 +152,12 @@ def create_certificate(keyout=None, certout=None):
     #san.append('DNS:{0}'.format(longname))
     san = ','.join(san)
     sslcfg = get_openssl_conf_location()
-    tmpconfig = tempfile.mktemp()
-    extconfig = tempfile.mktemp()
-    csrout = tempfile.mktemp()
+    tmphdl, tmpconfig = tempfile.mkstemp()
+    os.close(tmphdl)
+    tmphdl, extconfig = tempfile.mkstemp()
+    os.close(tmphdl)
+    tmphdl, csrout = tempfile.mkstemp()
+    os.close(tmphdl)
     shutil.copy2(sslcfg, tmpconfig)
     serialnum = '0x' + ''.join(['{:02x}'.format(x) for x in bytearray(os.urandom(20))])
     try:

confluent_server/confluent/collective/manager.py

@@ -71,7 +71,7 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
              'subsystem': 'collective'})
     try:
         remote = connect_to_collective(cert, leader, remote)
-    except socket.error as e:
+    except Exception as e:
         log.log({'error': 'Collective connection attempt to {0} failed: {1}'
                           ''.format(leader, str(e)),
                  'subsystem': 'collective'})
@@ -237,7 +237,7 @@ def get_myname():
             mycachedname[1] = time.time()
             return mycachedname[0]
     except IOError:
-        myname = socket.gethostname()
+        myname = socket.gethostname().split('.')[0]
         with open('/etc/confluent/cfg/myname', 'w') as f:
             f.write(myname)
         mycachedname[0] = myname

confluent_server/confluent/config/configmanager.py

@@ -546,15 +546,17 @@ def _load_dict_from_dbm(dpath, tdb):
             if elem not in currdict:
                 currdict[elem] = {}
             currdict = currdict[elem]
+        # Pickle is used as the first choice. It is a local self-owned file
+        # and thus not a significant security risk
         try:
             for tk in dbe.keys():
                 tks = confluent.util.stringify(tk)
-                currdict[tks] = cPickle.loads(dbe[tk])
+                currdict[tks] = cPickle.loads(dbe[tk]) # nosec
         except AttributeError:
             tk = dbe.firstkey()
             while tk != None:
                 tks = confluent.util.stringify(tk)
-                currdict[tks] = cPickle.loads(dbe[tk])
+                currdict[tks] = cPickle.loads(dbe[tk]) # nosec
                 tk = dbe.nextkey(tk)
     except dbm.error:
         return
@@ -1795,7 +1797,7 @@ class ConfigManager(object):
                         group))
             if not autocreate and group not in self._cfgstore['nodegroups']:
                 raise ValueError("{0} group does not exist".format(group))
-            for attr in attribmap[group]:
+            for attr in list(attribmap[group]):
                 # first do a pass to normalize out any aliased attribute names
                 if attr in _attraliases:
                     newattr = _attraliases[attr]
@@ -1822,6 +1824,13 @@ class ConfigManager(object):
                         if attribmap[group][attr].get('prepend', False):
                             newnodes = noderange.NodeRange(attribmap[group][attr][
                                 'prepend'], config=self).nodes
+                            pendingnodes = set([])
+                            for newnode in newnodes:
+                                if newnode in currnodes:
+                                    raise ValueError('{0} is already in group {1}'.format(newnode, group))
+                                if newnode in pendingnodes:
+                                    raise ValueError('{0} is listed multiple times')
+                                pendingnodes.add(newnode)
                             attribmap[group][attr] = list(
                                 newnodes) + currnodes
                         elif attribmap[group][attr].get('remove', False):
@@ -2243,6 +2252,13 @@ class ConfigManager(object):
                         if attribmap[node]['groups'].get('prepend', False):
                             newgroups = attribmap[node]['groups'][
                                 'prepend'].split(',')
+                            pendinggroups = set([])
+                            for newgroup in newgroups:
+                                if newgroup in currgroups:
+                                    raise ValueError('Node {0} is already in group {1}'.format(node, newgroup))
+                                if newgroup in pendinggroups:
+                                    raise ValueError('Group {0} has been specified multiple times'.format(newgroup))
+                                pendinggroups.add(newgroup)
                             attribmap[node]['groups'] = newgroups + currgroups
                         elif attribmap[node]['groups'].get('remove', False):
                             delgroups = attribmap[node]['groups'][

confluent_server/confluent/consoleserver.py

@@ -126,6 +126,7 @@ class ConsoleHandler(object):
                                   'collective.manager'))
 
     def __init__(self, node, configmanager, width=80, height=24):
+        self.termprefix = 'c_'
         self.clearpending = False
         self.clearerror = False
         self.initsize = (width, height)
@@ -190,12 +191,13 @@ class ConsoleHandler(object):
         if self.pendingbytes is not None:
             self.pendingbytes += data
         self.pendingbytes = b''
+        nodeid = self.termprefix + self.node
         try:
-            send_output(self.node, data)
+            send_output(nodeid, data)
             data = self.pendingbytes
             self.pendingbytes = None
             if data:
-                send_output(self.node, data)
+                send_output(nodeid, data)
         except Exception:
             _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                           event=log.Events.stacktrace)
@@ -547,7 +549,8 @@ class ConsoleHandler(object):
             'connectstate': self.connectstate,
             'clientcount': len(self.livesessions),
         }
-        retdata = get_buffer_output(self.node)
+        nodeid = self.termprefix + self.node
+        retdata = get_buffer_output(nodeid)
         return retdata, connstate
 
     def write(self, data):

confluent_server/confluent/core.py

@@ -158,7 +158,7 @@ def _merge_dict(original, custom):
 
 rootcollections = ['deployment/', 'discovery/', 'events/', 'networking/',
                    'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' ,
-                   'users/', 'version']
+                   'users/', 'uuid', 'version']
 
 
 class PluginRoute(object):
@@ -1216,6 +1216,8 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
             configmanager, inputdata, operation, pathcomponents)
     elif pathcomponents[0] == 'version':
         return (msg.Attributes(kv={'version': confluent.__version__}),)
+    elif pathcomponents[0] == 'uuid':
+        return (msg.Attributes(kv={'uuid': cfm.get_global('confluent_uuid')}),)
     elif pathcomponents[0] == 'usergroups':
         # TODO: when non-administrator accounts exist,
         # they must only be allowed to see their own user

confluent_server/confluent/discovery/core.py

@@ -106,7 +106,7 @@ class nesteddict(dict):
 nodehandlers = {
     'service:lenovo-smm': smm,
     'service:lenovo-smm2': smm,
-    'service:management-hardware.Lenovo:lenovo-xclarity-controller': xcc,
+    'lenovo-xcc': xcc,
     'service:management-hardware.IBM:integrated-management-module2': imm,
     'pxe-client': pxeh,
     'onie-switch': None,
@@ -122,7 +122,7 @@ servicenames = {
     'cumulus-switch': 'cumulus-switch',
     'service:lenovo-smm': 'lenovo-smm',
     'service:lenovo-smm2': 'lenovo-smm2',
-    'service:management-hardware.Lenovo:lenovo-xclarity-controller': 'lenovo-xcc',
+    'lenovo-xcc': 'lenovo-xcc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
     'service:thinkagile-storage': 'thinkagile-storagebmc',
@@ -135,7 +135,7 @@ servicebyname = {
     'cumulus-switch': 'cumulus-switch',
     'lenovo-smm': 'service:lenovo-smm',
     'lenovo-smm2': 'service:lenovo-smm2',
-    'lenovo-xcc': 'service:management-hardware.Lenovo:lenovo-xclarity-controller',
+    'lenovo-xcc': 'lenovo-xcc',
     'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2',
     'lenovo-switch': 'service:io-device.Lenovo:management-module',
     'thinkagile-storage': 'service:thinkagile-storagebmc',
@@ -635,18 +635,6 @@ def detected(info):
         eventlet.spawn_after(10, info['protocol'].fix_info, info,
                              safe_detected)
         return
-    try:
-        snum = info['attributes']['enclosure-serial-number'][0].strip()
-        if snum:
-            info['serialnumber'] = snum
-            known_serials[info['serialnumber']] = info
-    except (KeyError, IndexError):
-        pass
-    try:
-        info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
-        known_services[service].add(info['modelnumber'])
-    except (KeyError, IndexError):
-        pass
     if info['hwaddr'] in known_info and 'addresses' in info:
         # we should tee these up for parsing when an enclosure comes up
         # also when switch config parameters change, should discard
@@ -678,6 +666,22 @@ def detected(info):
     if handler:
         handler = handler.NodeHandler(info, cfg)
         handler.scan()
+    try:
+        if 'modelnumber' not in info:
+            info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
+    except (KeyError, IndexError):
+        pass
+    if 'modelnumber' in info:
+        known_services[service].add(info['modelnumber'])
+    try:
+        if 'serialnumber' not in info:
+            snum = info['attributes']['enclosure-serial-number'][0].strip()
+            if snum:
+                info['serialnumber'] = snum
+    except (KeyError, IndexError):
+        pass
+    if 'serialnumber' in info:
+        known_serials[info['serialnumber']] = info
     uuid = info.get('uuid', None)
     if uuid_is_valid(uuid):
         known_uuids[uuid][info['hwaddr']] = info
@@ -1188,7 +1192,7 @@ def discover_node(cfg, handler, info, nodename, manual):
         return True
     if info['handler'] == pxeh:
         olduuid = dp.get(nodename, {}).get('id.uuid', {}).get(
-            'value', None)
+            'value', '')
         if olduuid.lower() != info['uuid']:
             log.log({'info': 'Detected {0}, but discovery.policy is not set to a '
                             'value allowing discovery (open, permissive, or pxe)'.format(
@@ -1340,7 +1344,7 @@ def start_detection():
     if rechecker is None:
         rechecktime = util.monotonic_time() + 900
         rechecker = eventlet.spawn_after(900, _periodic_recheck, cfg)
-    eventlet.spawn_n(ssdp.snoop, None, None, ssdp, get_node_by_uuid_or_mac)
+    eventlet.spawn_n(ssdp.snoop, safe_detected, None, ssdp, get_node_by_uuid_or_mac)
 
 def stop_autosense():
     for watcher in list(autosensors):

confluent_server/confluent/discovery/handlers/smm.py

@@ -36,7 +36,9 @@ def fromstring(inputdata):
         cmpstr = '!entity'
     if cmpstr in inputdata.lower():
         raise Exception('!ENTITY not supported in this interface')
-    return rfromstring(inputdata)
+    # The measures above should filter out the risky facets of xml
+    # We don't need sophisticated feature support
+    return rfromstring(inputdata)  # nosec
 
 def fixuuid(baduuid):
     # SMM dumps it out in hex

confluent_server/confluent/discovery/handlers/tsm.py

@@ -50,7 +50,7 @@ class NodeHandler(generic.NodeHandler):
         i = c.grab_json_response('/redfish/v1/')
         uuid = i.get('UUID', None)
         if uuid:
-            self.info['uuid'] = uuid
+            self.info['uuid'] = uuid.lower()
 
     def validate_cert(self, certificate):
         # broadly speaking, merely checks consistency moment to moment,

confluent_server/confluent/discovery/handlers/xcc.py

@@ -30,15 +30,6 @@ import struct
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
 
-def fixup_uuid(uuidprop):
-    baduuid = ''.join(uuidprop.split())
-    uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
-    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
-    a = util.stringify(a)
-    uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
-    return '-'.join(uuid).upper()
-
-
 class LockedUserException(Exception):
     pass
 
@@ -63,6 +54,9 @@ class NodeHandler(immhandler.NodeHandler):
         # This is not adequate for being satisfied
         return bool(info.get('attributes', {}))
 
+    def probe(self):
+        return None
+
     def scan(self):
         c = webclient.SecureHTTPConnection(self.ipaddr, 443,
             verifycallback=self.validate_cert)
@@ -70,7 +64,40 @@ class NodeHandler(immhandler.NodeHandler):
         modelname = i.get('items', [{}])[0].get('machine_name', None)
         if modelname:
             self.info['modelname'] = modelname
-        super(NodeHandler, self).scan()
+        for attrname in list(self.info.get('attributes', {})):
+            val = self.info['attributes'][attrname]
+            if '-uuid' == attrname[-5:] and len(val) == 32:
+                val = val.lower()
+                self.info['attributes'][attrname] = '-'.join([val[:8], val[8:12], val[12:16], val[16:20], val[20:]])
+        attrs = self.info.get('attributes', {})
+        room = attrs.get('room-id', None)
+        if room:
+            self.info['room'] = room
+        rack = attrs.get('rack-id', None)
+        if rack:
+            self.info['rack'] = rack
+        name = attrs.get('name', None)
+        if name:
+            self.info['hostname'] = name
+        unumber = attrs.get('lowest-u', None)
+        if unumber:
+            self.info['u'] = unumber
+        location = attrs.get('location', None)
+        if location:
+            self.info['location'] = location
+        mtm = attrs.get('enclosure-machinetype-model', None)
+        if mtm:
+            self.info['modelnumber'] = mtm.strip()
+        sn = attrs.get('enclosure-serial-number', None)
+        if sn:
+            self.info['serialnumber'] = sn.strip()
+        if attrs.get('enclosure-form-factor', None) == 'dense-computing':
+            encuuid = attrs.get('chassis-uuid', None)
+            if encuuid:
+                self.info['enclosure.uuid'] = encuuid
+            slot = int(attrs.get('slot', 0))
+            if slot != 0:
+                self.info['enclosure.bay'] = slot
 
     def preconfig(self, possiblenode):
         self.tmpnodename = possiblenode
@@ -78,16 +105,18 @@ class NodeHandler(immhandler.NodeHandler):
         if ff not in ('dense-computing', [u'dense-computing']):
             # skip preconfig for non-SD530 servers
             return
-        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [''])[0]
-        if not currfirm.startswith('TEI'):
+        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [{}])[0]
+        if not currfirm.get('build', '').startswith('TEI'):
             return
         self.trieddefault = None  # Reset state on a preconfig attempt
         # attempt to enable SMM
         #it's normal to get a 'not supported' (193) for systems without an SMM
         # need to branch on 3.00+ firmware
-        currfirm = currfirm.split(':')
-        if len(currfirm) > 1:
-            currfirm = float(currfirm[1])
+        currfirm = currfirm.get('version', '0.0')
+        if currfirm:
+            currfirm = float(currfirm)
+        else:
+            currfirm = 0
         disableipmi = False
         if currfirm >= 3:
             # IPMI is disabled and we need it, also we need to go to *some* password
@@ -499,7 +528,7 @@ class NodeHandler(immhandler.NodeHandler):
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
             return
-        enclosureuuid = self.info.get('attributes', {}).get('chassis-uuid', [None])[0]
+        enclosureuuid = self.info.get('enclosure.uuid', None)
         if enclosureuuid:
             enclosureuuid = enclosureuuid.lower()
             em = self.configmanager.get_node_attributes(nodename,

confluent_server/confluent/discovery/protocols/pxe.py

@@ -345,7 +345,11 @@ def snoop(handler, protocol=None, nodeguess=None):
     net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
     net4.setsockopt(socket.IPPROTO_IP, IP_PKTINFO, 1)
-    net4.bind(('', 67))
+    try:
+        net4.bind(('', 67))
+    except Exception:
+        log.log({'error': 'Unable to bind DHCP server port, if using dnsmasq, specify bind-dynamic in dnsmasq.conf and restart dnsmasq and then confluent'})
+        return
     v6addr = socket.inet_pton(socket.AF_INET6, mcastv6addr)
     net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
     net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -490,6 +494,10 @@ def new_nodes(added, deleting, renamed, configmanager):
     configmanager.remove_watcher(attribwatcher)
     alldeleting = set(deleting) | set(renamed)
     clear_nodes(alldeleting)
+    alladding = set(added)
+    for oldname in renamed:
+        alladding.add(renamed[oldname])
+    remap_nodes(alladding, configmanager)
     attribwatcher = configmanager.watch_attributes(configmanager.list_nodes(),
                                                    ('id.uuid', 'net.*hwaddr'), remap_nodes)
 

confluent_server/confluent/discovery/protocols/slp.py

@@ -31,7 +31,7 @@ _slp_services = set([
     'service:lenovo-smm2',
     'service:ipmi',
     'service:lighttpd',
-    'service:management-hardware.Lenovo:lenovo-xclarity-controller',
+    #'service:management-hardware.Lenovo:lenovo-xclarity-controller',
     'service:management-hardware.IBM:chassis-management-module',
     'service:management-hardware.Lenovo:chassis-management-module',
     'service:io-device.Lenovo:management-module',
@@ -534,7 +534,10 @@ def process_peer(newmacs, known_peers, peerbymacaddress, peer):
     if mac in peerbymacaddress:
         peerbymacaddress[mac]['addresses'].append(peer)
     else:
-        q = query_srvtypes(peer)
+        try:
+            q = query_srvtypes(peer)
+        except Exception as e:
+            q = None
         if not q or not q[0]:
             # SLP might have started and not ready yet
             # ignore for now

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2017 Lenovo
+# Copyright 2017-2022 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,14 +39,12 @@ import eventlet
 import eventlet.green.select as select
 import eventlet.green.socket as socket
 import eventlet.greenpool as gp
+import os
 import time
-try:
-    from eventlet.green.urllib.request import urlopen
-except (ImportError, AssertionError):
-    from eventlet.green.urllib2 import urlopen
 import struct
 import traceback
 
+webclient = eventlet.import_patched('pyghmi.util.webclient')
 mcastv4addr = '239.255.255.250'
 mcastv6addr = 'ff02::c'
 
@@ -80,17 +78,14 @@ def scan(services, target=None):
             yield rply
 
 
-def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers):
-    known_peers.add(peer)
-    newmacs.add(mac)
-    if mac in peerbymacaddress:
+def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler):
+    if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']:
         peerbymacaddress[mac]['addresses'].append(peer)
     else:
-        peerbymacaddress[mac] = {
+        peerdata = {
             'hwaddr': mac,
             'addresses': [peer],
         }
-        peerdata = peerbymacaddress[mac]
         for headline in rsp[1:]:
             if not headline:
                 continue
@@ -99,12 +94,27 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha
             header = header.strip()
             value = value.strip()
             if header == 'NT':
-                peerdata['service'] = value
+                if 'redfish-rest' not in value:
+                    return
             elif header == 'NTS':
                 if value == 'ssdp:byebye':
-                    machandlers[mac] = byehandler
-                elif value == 'ssdp:alive':
-                    machandlers[mac] = None # handler
+                    handler = byehandler
+                elif value != 'ssdp:alive':
+                    handler = None
+            elif header == 'AL':
+                if not value.endswith('/redfish/v1/'):
+                    return
+            elif header == 'LOCATION':
+                if not value.endswith('/DeviceDescription.json'):
+                    return
+        if handler:
+            retdata = check_fish(('/DeviceDescription.json', peerdata))
+            if retdata:
+                known_peers.add(peer)
+                newmacs.add(mac)
+                peerbymacaddress[mac] = retdata
+                machandlers[mac] = handler
+
 
 def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
     """Watch for SSDP notify messages
@@ -123,6 +133,11 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
     # dabbling in multicast wizardry here, such sockets can cause big problems,
     # so we will have two distinct sockets
     tracelog = log.Logger('trace')
+    try:
+        active_scan(handler, protocol)
+    except Exception as e:
+        tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                    event=log.Events.stacktrace)
     known_peers = set([])
     net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
     net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
@@ -167,13 +182,14 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                             continue
                         mac = neighutil.get_hwaddr(peer[0])
                         if not mac:
+                            probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
                             try:
-                                s.sendto(b'\x00', peer)
+                                s.sendto(b'\x00', probepeer)
                             except Exception:
                                 continue
                             deferrednotifies.append((peer, rsp))
                             continue
-                        _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers)
+                        _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
                     elif method == b'M-SEARCH':
                         if not uuidlookup:
                             continue
@@ -244,10 +260,12 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                     r = r[0]
             if deferrednotifies:
                 eventlet.sleep(2.2)
+            for peerrsp in deferrednotifies:
+                peer, rsp = peerrsp
                 mac = neighutil.get_hwaddr(peer[0])
                 if not mac:
                     continue
-                _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers)
+                _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
             for mac in newmacs:
                 thehandler = machandlers.get(mac, None)
                 if thehandler:
@@ -257,6 +275,12 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                              event=log.Events.stacktrace)
 
 
+def _get_svrip(peerdata):
+    for addr in peerdata['addresses']:
+        if addr[0].startswith('fe80::'):
+            return addr[0]
+    return peerdata['addresses'][0][0]
+
 def _find_service(service, target):
     net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
@@ -299,7 +323,11 @@ def _find_service(service, target):
             msg = smsg.format(mcastv4addr, service)
             if not isinstance(msg, bytes):
                 msg = msg.encode('utf8')
-            net4.sendto(msg, (mcastv4addr, 1900))
+            try:
+                net4.sendto(msg, (mcastv4addr, 1900))
+            except socket.error as se:
+                if se.errno != 101:
+                    raise
             msg = smsg.format(bcast, service)
             if not isinstance(msg, bytes):
                 msg = msg.encode('utf8')
@@ -314,8 +342,9 @@ def _find_service(service, target):
         for s in r:
             (rsp, peer) = s.recvfrom(9000)
             if not neighutil.get_hwaddr(peer[0]):
+                probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
                 try:
-                    s.sendto(b'\x00', peer)
+                    s.sendto(b'\x00', probepeer)
                 except Exception:
                     continue
                 deferparse.append((rsp, peer))
@@ -333,25 +362,49 @@ def _find_service(service, target):
     querypool = gp.GreenPool()
     pooltargs = []
     for nid in peerdata:
-        for url in peerdata[nid].get('urls', ()):
-            if url.endswith('/desc.tmpl'):
-                pooltargs.append((url, peerdata[nid]))
-    for pi in querypool.imap(check_cpstorage, pooltargs):
+        if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
+            continue
+        if '/DeviceDescription.json' in peerdata[nid]['urls']:
+            pooltargs.append(('/DeviceDescription.json', peerdata[nid]))
+        # For now, don't interrogate generic redfish bmcs
+        # This is due to a need to deduplicate from some supported SLP
+        # targets (IMM, TSM, others)
+        # activate this else once the core filters/merges duplicate uuid
+        # or we drop support for those devices
+        #else:
+        #    pooltargs.append(('/redfish/v1/', peerdata[nid]))
+    for pi in querypool.imap(check_fish, pooltargs):
         if pi is not None:
             yield pi
 
-def check_cpstorage(urldata):
+def check_fish(urldata):
     url, data = urldata
     try:
-        info = urlopen(url, timeout=1).read()
-        if b'<friendlyName>Athena</friendlyName>' in info:
-            data['services'] = ['service:thinkagile-storage']
+        wc = webclient.SecureHTTPConnection(_get_svrip(data), 443, verifycallback=lambda x: True)
+        peerinfo = wc.grab_json_response(url)
+    except socket.error:
+        return None
+    if url == '/DeviceDescription.json':
+        try:
+            peerinfo = peerinfo[0]
+            myuuid = peerinfo['node-uuid'].lower()
+            if '-' not in myuuid:
+                myuuid = '-'.join([myuuid[:8], myuuid[8:12], myuuid[12:16], myuuid[16:20], myuuid[20:]])
+            data['uuid'] = myuuid
+            data['attributes'] = peerinfo
+            data['services'] = ['lenovo-xcc']
+            return data
+        except (IndexError, KeyError):
+            return None
+            url = '/redfish/v1/'
+            peerinfo = wc.grab_json_response('/redfish/v1/')
+    if url == '/redfish/v1/':
+        if 'UUID' in peerinfo:
+            data['services'] = ['service:redfish-bmc']
+            data['uuid'] = peerinfo['UUID'].lower()
             return data
-    except Exception:
-        pass
     return None
 
-
 def _parse_ssdp(peer, rsp, peerdata):
     nid = peer[0]
     mac = None
@@ -378,9 +431,11 @@ def _parse_ssdp(peer, rsp, peerdata):
             if not headline:
                 continue
             header, _, value = headline.partition(b':')
-            header = header.strip()
-            value = value.strip()
+            header = header.strip().decode('utf8')
+            value = value.strip().decode('utf8')
             if header == 'AL' or header == 'LOCATION':
+                value = value[value.index('://')+3:]
+                value = value[value.index('/'):]
                 if 'urls' not in peerdatum:
                     peerdatum['urls'] = [value]
                 elif value not in peerdatum['urls']:

confluent_server/confluent/firmwaremanager.py

@@ -22,6 +22,7 @@ import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
 import eventlet
+import io
 import os
 import pwd
 import socket
@@ -32,63 +33,71 @@ uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
 _tracelog = None
-filecontentbyname = {}
+sharedfiles = {}
 
 
 def execupdate(handler, filename, updateobj, type, owner, node, datfile):
     global _tracelog
-    if type != 'ffdc' and not datfile:
-        errstr = False
-        if not os.path.exists(filename):
-            errstr =  '{0} does not appear to exist on {1}, or is in a directory with permissions forbidding confluent user/group access'.format(
-                filename, socket.gethostname())
-        elif not os.access(filename, os.R_OK):
-            errstr =  '{0} is not readable by confluent on {1} (ensure confluent user or group can access file and parent directories)'.format(
-                filename, socket.gethostname())
-        if errstr:
-            updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
-                                    'detail': errstr})
-            return
-    if type == 'ffdc' and os.path.isdir(filename):
-        filename += '/' + node
-    if 'type' == 'ffdc':
-        errstr = False
-        if os.path.exists(filename):
-            errstr = '{0} already exists on {1}, cannot overwrite'.format(
-                filename, socket.gethostname())
-        elif not os.access(os.path.dirname(filename), os.W_OK):
-            errstr = '{0} directory not writable by confluent user/group on {1}, check the directory and parent directory ownership and permissions'.format(filename, socket.gethostname())
-        if errstr:
-            updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
-                                    'detail': errstr})
-            return
     try:
-        if type == 'firmware':
-            completion = handler(filename, progress=updateobj.handle_progress,
-                                 data=datfile, bank=updateobj.bank)
-        else:
-            completion = handler(filename, progress=updateobj.handle_progress,
-                                 data=datfile)
-        if type == 'ffdc' and completion:
-            filename = completion
-            completion = None
-        if completion is None:
-            completion = 'complete'
-        if owner:
-            pwent = pwd.getpwnam(owner)
-            os.chown(filename, pwent.pw_uid, pwent.pw_gid)
-        updateobj.handle_progress({'phase': completion, 'progress': 100.0})
-    except exc.PubkeyInvalid as pi:
-        errstr = 'Certificate mismatch detected, does not match value in ' \
-                 'attribute {0}'.format(pi.attrname)
-        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
-                                   'detail': errstr})
-    except Exception as e:
-        if _tracelog is None:
-            _tracelog = log.Logger('trace')
-        _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace)
-        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
-                                   'detail': str(e)})
+        if type != 'ffdc' and not datfile:
+            errstr = False
+            if not os.path.exists(filename):
+                errstr =  '{0} does not appear to exist on {1}, or is in a directory with permissions forbidding confluent user/group access'.format(
+                    filename, socket.gethostname())
+            elif not os.access(filename, os.R_OK):
+                errstr =  '{0} is not readable by confluent on {1} (ensure confluent user or group can access file and parent directories)'.format(
+                    filename, socket.gethostname())
+            if errstr:
+                updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                        'detail': errstr})
+                return
+        if type == 'ffdc' and os.path.isdir(filename):
+            filename += '/' + node
+        if 'type' == 'ffdc':
+            errstr = False
+            if os.path.exists(filename):
+                errstr = '{0} already exists on {1}, cannot overwrite'.format(
+                    filename, socket.gethostname())
+            elif not os.access(os.path.dirname(filename), os.W_OK):
+                errstr = '{0} directory not writable by confluent user/group on {1}, check the directory and parent directory ownership and permissions'.format(filename, socket.gethostname())
+            if errstr:
+                updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                        'detail': errstr})
+                return
+        try:
+            if type == 'firmware':
+                completion = handler(filename, progress=updateobj.handle_progress,
+                                    data=datfile, bank=updateobj.bank)
+            else:
+                completion = handler(filename, progress=updateobj.handle_progress,
+                                    data=datfile)
+            if type == 'ffdc' and completion:
+                filename = completion
+                completion = None
+            if completion is None:
+                completion = 'complete'
+            if owner:
+                pwent = pwd.getpwnam(owner)
+                os.chown(filename, pwent.pw_uid, pwent.pw_gid)
+            updateobj.handle_progress({'phase': completion, 'progress': 100.0})
+        except exc.PubkeyInvalid as pi:
+            errstr = 'Certificate mismatch detected, does not match value in ' \
+                    'attribute {0}'.format(pi.attrname)
+            updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                    'detail': errstr})
+        except Exception as e:
+            if _tracelog is None:
+                _tracelog = log.Logger('trace')
+            _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace)
+            updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                    'detail': str(e)})
+    finally:
+        if filename in sharedfiles:
+            if sharedfiles[filename][0] == 1:
+                del sharedfiles[filename]
+            else:
+                sharedfiles[filename][0] -= 1
+
 
 class Updater(object):
     def __init__(self, node, handler, filename, tenant=None, name=None,
@@ -99,8 +108,14 @@ class Updater(object):
         self.detail = ''
         self.percent = 0.0
         if configmanager and filename in configmanager.clientfiles:
-            cf = configmanager.clientfiles[filename]
-            datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+            if filename in sharedfiles:
+                sharedfiles[filename][0] += 1
+            else:
+                cf = configmanager.clientfiles[filename]
+                datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+                sharedfiles[filename] = [1, datfile.read()]
+                datfile.close()
+            datfile = io.BytesIO(sharedfiles[filename][1])
         else:
             datfile = None
         self.datfile = datfile

confluent_server/confluent/main.py

@@ -272,6 +272,8 @@ def run(args):
     if not confluentuuid:
         confluentuuid = str(uuid.uuid4())
         configmanager.set_global('confluent_uuid', confluentuuid)
+    if not configmanager._masterkey:
+        configmanager.init_masterkey()
     if dbgif:
         oumask = os.umask(0o077)
         try:

confluent_server/confluent/neighutil.py

@@ -22,6 +22,11 @@ import eventlet.semaphore as semaphore
 import eventlet.green.socket as socket
 import struct
 
+
+def msg_align(len):
+    return (len + 3) & ~3
+
+
 neightable = {}
 neightime = 0
 
@@ -65,12 +70,12 @@ def _update_neigh():
                                     curraddr = curraddr[12:]
                             elif rtatyp == 1:  # ip address
                                 currip = rta[4:rtalen].tobytes()
-                            rta = rta[rtalen:]
+                            rta = rta[msg_align(rtalen):]
                             if not rtalen:
                                 break
                         if curraddr and currip:
                             neightable[currip] = curraddr
-                v = v[length:]
+                v = v[msg_align(length):]
     finally:
         s.close()
 

confluent_server/confluent/netutil.py

@@ -25,6 +25,8 @@ import eventlet.support.greendns
 import os
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
+def msg_align(len):
+    return (len + 3) & ~3
 
 def mask_to_cidr(mask):
     maskn = socket.inet_pton(socket.AF_INET, mask)
@@ -571,7 +573,7 @@ def get_my_addresses(idx=0, family=0, matchlla=None):
                                 break
                             if rta[4:rtalen].tobytes() == matchlla:
                                 return get_my_addresses(idx=ridx)
-                            rta = rta[rtalen:]
+                            rta = rta[msg_align(rtalen):]
                 elif (ridx == idx or not idx) and scope == 0:
                     rta = v[nlhdrsz+ifaddrsz:length]
                     while len(rta):
@@ -580,8 +582,8 @@ def get_my_addresses(idx=0, family=0, matchlla=None):
                             break
                         if rtatyp == 1:
                             addrs.append((fam, rta[4:rtalen].tobytes(), plen, ridx))
-                        rta = rta[rtalen:]
-            v = v[length:]
+                        rta = rta[msg_align(rtalen):]
+            v = v[msg_align(length):]
     return addrs
 
 

confluent_server/confluent/networking/macmap.py

@@ -170,9 +170,17 @@ def _affluent_map_switch(args):
                 if mac in _nodesbymac and _nodesbymac[mac][0] != nodename:
                     # For example, listed on both a real edge port
                     # and by accident a trunk port
-                    log.log({'error': '{0} and {1} described by ambiguous'
-                                    ' switch topology values'.format(
-                                        nodename, _nodesbymac[mac][0])})
+                    onummacs = _nodesbymac[mac][1]
+                    onode = _nodesbymac[mac][0]
+                    if onode:
+                        errstr = 'Mac address {2} may match either {0} or {1} according to net.*switch* attributes.'.format(nodename, onode, mac)
+                        if onummacs > 2 or nummacs > 2:
+                            errstr += ' ({0} may match a switch trunk)'.format(nodename if nummacs > onummacs else onode)
+                    else:
+                        errstr = 'Mac address {1} may match either {0} or a node previously reported as ambiguous according to net.*switch* attributes.'.format(nodename, mac)
+                        if nummacs > 2:
+                            errstr += ' ({0} may match a switch trunk)'.format(nodename)
+                    log.log({'error': errstr})
                     _nodesbymac[mac] = (None, None)
                 else:
                     _nodesbymac[mac] = (nodename, nummacs)
@@ -306,9 +314,19 @@ def _map_switch_backend(args):
             if mac in _nodesbymac and _nodesbymac[mac][0] != nodename:
                 # For example, listed on both a real edge port
                 # and by accident a trunk port
-                log.log({'error': '{0} and {1} described by ambiguous'
-                                  ' switch topology values'.format(
-                                      nodename, _nodesbymac[mac][0])})
+                nummacs = maccounts[ifname]
+                onummacs = _nodesbymac[mac][1]
+                onode = _nodesbymac[mac][0]
+                if onode:
+                    errstr = 'Mac address {2} may match either {0} or {1} according to net.*switch* attributes.'.format(nodename, onode, mac)
+                    if onummacs > 2 or nummacs > 2:
+                        errstr += ' ({0} may match a link between switches)'.format(nodename if nummacs > onummacs else onode)
+                else:
+                    errstr = 'Mac address {1} may match either {0} or a node previously reported as ambiguous according to net.*switch* attributes.'.format(nodename, mac)
+                    if nummacs > 2:
+                        errstr += ' ({0} may match a link between switches)'.format(nodename)
+                log.log({'error': errstr})
+                
                 _nodesbymac[mac] = (None, None)
             else:
                 _nodesbymac[mac] = (nodename, maccounts[ifname])

confluent_server/confluent/noderange.py

@@ -196,6 +196,7 @@ class NodeRange(object):
                 leftnum = int(leftbits[idx])
                 rightnum = int(rightbits[idx])
                 if leftnum > rightnum:
+                    return self.failorreturn(seqrange)
                     width = len(rightbits[idx])
                     minnum = rightnum
                     maxnum = leftnum + 1  # range goes to n-1...

confluent_server/confluent/osimage.py

@@ -212,7 +212,8 @@ def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist
                 write_data_block(write_p, buff, size, offset)
             write_finish_entry(write_p)
             if os.path.isdir(str(entry)):
-                os.chmod(str(entry), 0o755)
+                # This directory must be world accessible for web server
+                os.chmod(str(entry), 0o755)  # nosec
             else:
                 os.chmod(str(entry), 0o644)
     if callback:
@@ -484,6 +485,10 @@ def check_rhel(isoinfo):
             ver = entry.split('-')[2]
             arch = entry.split('.')[-2]
             break
+        elif 'redhat-release-9' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            break
     else:
         if '.discinfo' in isoinfo[1]:
             prodinfo = isoinfo[1]['.discinfo']
@@ -567,7 +572,10 @@ def import_image(filename, callback, backend=False, mfd=None):
     if identity.get('subname', None):
         targpath += '/' + identity['subname']
     targpath = '/var/lib/confluent/distributions/' + targpath
-    os.makedirs(targpath, 0o755)
+    try:
+        os.makedirs(targpath, 0o755)
+    except Exception as e:
+        sys.stdout.write('ERROR:{0}\r'.format(str(e)))
     filename = os.path.abspath(filename)
     identity['importedfile'] = filename
     os.chdir(targpath)
@@ -716,6 +724,7 @@ class MediaImporter(object):
             raise Exception('{0} already exists'.format(self.targpath))
         self.filename = os.path.abspath(media)
         self.medfile = medfile
+        self.error = ''
         self.importer = eventlet.spawn(self.importmedia)
 
     def stop(self):
@@ -724,7 +733,7 @@ class MediaImporter(object):
 
     @property
     def progress(self):
-        return {'phase': self.phase, 'progress': self.percent, 'profiles': self.profiles}
+        return {'phase': self.phase, 'progress': self.percent, 'profiles': self.profiles, 'error': self.error}
 
     def importmedia(self):
         os.environ['PYTHONPATH'] = ':'.join(sys.path)
@@ -738,24 +747,44 @@ class MediaImporter(object):
         while wkr.poll() is None:
             currline += wkr.stdout.read(1)
             if b'\r' in currline:
-                val = currline.split(b'%')[0].strip()
-                if val:
-                    self.percent = float(val)
+                if b'%' in currline:
+                    val = currline.split(b'%')[0].strip()
+                    if val:
+                        self.percent = float(val)
+                elif b'ERROR:' in currline:
+                    self.error = currline.replace(b'ERROR:', b'')
+                    if not isinstance(self.error, str):
+                        self.error = self.error.decode('utf8')
+                    self.phase = 'error'
+                    self.percent = 100.0
+                    return
                 currline = b''
         a = wkr.stdout.read(1)
         while a:
             currline += a
             if b'\r' in currline:
-                val = currline.split(b'%')[0].strip()
-                if val:
-                    self.percent = float(val)
+                if b'%' in currline:
+                    val = currline.split(b'%')[0].strip()
+                    if val:
+                        self.percent = float(val)
+                elif b'ERROR:' in currline:
+                    self.error = currline.replace(b'ERROR:', b'')
+                    if not isinstance(self.error, str):
+                        self.error = self.error.decode('utf8')
+                    self.phase = 'error'
+                    return
             currline = b''
             a = wkr.stdout.read(1)
         if self.oscategory:
             defprofile = '/opt/confluent/lib/osdeploy/{0}'.format(
                 self.oscategory)
-            generate_stock_profiles(defprofile, self.distpath, self.targpath,
-                                    self.osname, self.profiles)
+            try:
+                generate_stock_profiles(defprofile, self.distpath, self.targpath,
+                                        self.osname, self.profiles)
+            except Exception as e:
+                self.phase = 'error'
+                self.error = str(e)
+                raise
         self.phase = 'complete'
         self.percent = 100.0
 

confluent_server/confluent/pam.py

@@ -242,10 +242,10 @@ if __name__ == "__main__":
         readline.set_pre_input_hook(hook)
 
         if sys.version_info >= (3,):
-            result = input(prompt)
+            getinput = input
         else:
-            result = raw_input(prompt)
-
+            getinput = raw_input
+        result = getinput(prompt)
         readline.set_pre_input_hook()
         return result
 

confluent_server/confluent/plugins/configuration/attributes.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import ast
 import confluent.exceptions as exc
 import confluent.messages as msg
 import confluent.config.attributes as allattributes
@@ -188,6 +189,12 @@ def update_nodegroup(group, element, configmanager, inputdata):
         for attrib in inputdata.attribs:
             if inputdata.attribs[attrib] is None:
                 clearattribs.append(attrib)
+            else:
+                try:
+                    ast.parse(attrib)
+                except SyntaxError as e:
+                    markup = (e.text[:e.offset-1] + '-->' + e.text[e.offset-1] + '<--' + e.text[e.offset:]).strip()
+                    raise exc.InvalidArgumentException('Syntax error in attribute name: "{0}"'.format(markup))
         for attrib in clearattribs:
             del inputdata.attribs[attrib]
         if clearattribs:
@@ -211,7 +218,12 @@ def _expand_expression(nodes, configmanager, inputdata):
             pernodeexpressions[expanded[0]] = expanded[1]
         for node in util.natural_sort(pernodeexpressions):
             yield msg.KeyValueData({'value': pernodeexpressions[node]}, node)
-    except (SyntaxError, ValueError) as e:
+    except SyntaxError as e:
+        markup = (e.text[:e.offset-1] + '-->' + e.text[e.offset-1] + '<--' + e.text[e.offset:]).strip()
+        raise exc.InvalidArgumentException(
+            'Bad confluent expression syntax (must use "{{" and "}}" if not '
+            'desiring confluent expansion): ' + markup)
+    except ValueError as e:
         raise exc.InvalidArgumentException(
             'Bad confluent expression syntax (must use "{{" and "}}" if not '
             'desiring confluent expansion): ' + str(e))
@@ -261,7 +273,11 @@ def update_nodes(nodes, element, configmanager, inputdata):
             for attrib in list(updatenode):
                 if updatenode[attrib] is None:
                     del updatenode[attrib]
-                    if attrib in allattributes.node  or attrib.startswith('custom.') or attrib.startswith('net.'):
+                    if '*' in attrib:
+                        currnodeattrs = configmanager.get_node_attributes(node, attrib)
+                        for matchattrib in currnodeattrs.get(node, {}):
+                            clearattribs.append(matchattrib)
+                    elif attrib in allattributes.node  or attrib.startswith('custom.') or attrib.startswith('net.'):
                         clearattribs.append(attrib)
                     else:
                         foundattrib = False
@@ -271,6 +287,17 @@ def update_nodes(nodes, element, configmanager, inputdata):
                                 foundattrib = True
                         if not foundattrib:
                             raise exc.InvalidArgumentException("No attribute matches '" + attrib + "' (try wildcard if trying to clear a group)")
+                elif '*' in attrib:
+                    currnodeattrs = configmanager.get_node_attributes(node, attrib)
+                    for matchattrib in currnodeattrs.get(node, {}):
+                        updatenode[matchattrib] = updatenode[attrib]
+                    del updatenode[attrib]
+                else:
+                    try:
+                        ast.parse(attrib)
+                    except SyntaxError as e:
+                        markup = (e.text[:e.offset-1] + '-->' + e.text[e.offset-1] + '<--' + e.text[e.offset:]).strip()
+                        raise exc.InvalidArgumentException('Syntax error in attribute name: "{0}"'.format(markup))
             if len(clearattribs) > 0:
                 configmanager.clear_node_attributes([node], clearattribs)
             updatedict[node] = updatenode

confluent_server/confluent/selfservice.py

@@ -25,7 +25,7 @@ currtzvintage = None
 def yamldump(input):
     return yaml.safe_dump(input, default_flow_style=False)
 
-def get_extra_names(nodename, cfg):
+def get_extra_names(nodename, cfg, myip=None):
     names = set([])
     dnsinfo = cfg.get_node_attributes(nodename, ('dns.*', 'net.*hostname'))
     dnsinfo = dnsinfo.get(nodename, {})
@@ -41,6 +41,17 @@ def get_extra_names(nodename, cfg):
                     names.add(currname)
                     if domain and domain not in currname:
                         names.add('{0}.{1}'.format(currname, domain))
+    if myip:
+        ncfgs = [netutil.get_nic_config(cfg, nodename, serverip=myip)]
+        fncfg = netutil.get_full_net_config(cfg, nodename, serverip=myip)
+        ncfgs.append(fncfg.get('default', {}))
+        for ent in fncfg.get('extranets', []):
+            ncfgs.append(fncfg['extranets'][ent])
+        for ncfg in ncfgs:
+            for nip in (ncfg.get('ipv4_address', None), ncfg.get('ipv6_address', None)):
+                if nip:
+                    nip = nip.split('/', 1)[0]
+                    names.add(nip)
     return names
 
 def handle_request(env, start_response):
@@ -70,6 +81,13 @@ def handle_request(env, start_response):
         return
     if ea.get(nodename, {}).get('deployment.apiarmed', {}).get('value', None) == 'once':
         cfg.set_node_attributes({nodename: {'deployment.apiarmed': ''}})
+    myip = env.get('HTTP_X_FORWARDED_HOST', None)
+    if myip and ']' in myip:
+        myip = myip.split(']', 1)[0]
+    elif myip:
+        myip = myip.split(':', 1)[0]
+    if myip:
+        myip = myip.replace('[', '').replace(']', '')
     retype = env.get('HTTP_ACCEPT', 'application/yaml')
     isgeneric = False
     if retype == '*/*':
@@ -118,12 +136,6 @@ def handle_request(env, start_response):
         start_response('200 OK', (('Conntent-Type', retype),))
         yield dumper(rsp)
     elif env['PATH_INFO'] == '/self/netcfg':
-        myip = env.get('HTTP_X_FORWARDED_HOST', None)
-        if ']' in myip:
-            myip = myip.split(']', 1)[0]
-        else:
-            myip = myip.split(':', 1)[0]
-        myip = myip.replace('[', '').replace(']', '')
         ncfg = netutil.get_full_net_config(cfg, nodename, myip)
         start_response('200 OK', (('Content-Type', retype),))
         yield dumper(ncfg)
@@ -137,12 +149,6 @@ def handle_request(env, start_response):
                     ifidx = int(nici.read())
             ncfg = netutil.get_nic_config(cfg, nodename, ifidx=ifidx)
         else:
-            myip = env.get('HTTP_X_FORWARDED_HOST', None)
-            if ']' in myip:
-                myip = myip.split(']', 1)[0]
-            else:
-                myip = myip.split(':', 1)[0]
-            myip = myip.replace('[', '').replace(']', '')
             ncfg = netutil.get_nic_config(cfg, nodename, serverip=myip)
         if env['PATH_INFO'] == '/self/deploycfg':
             for key in list(ncfg):
@@ -270,7 +276,7 @@ def handle_request(env, start_response):
             start_response('500 Unconfigured', ())
             yield 'CA is not configured on this system (run ...)'
             return
-        pals = get_extra_names(nodename, cfg)
+        pals = get_extra_names(nodename, cfg, myip)
         cert = sshutil.sign_host_key(reqbody, nodename, pals)
         start_response('200 OK', (('Content-Type', 'text/plain'),))
         yield cert

confluent_server/confluent/shellserver.py

@@ -37,6 +37,10 @@ class _ShellHandler(consoleserver.ConsoleHandler):
         # suppress logging through proving a stub 'log' function
         return
 
+    def feedbuffer(self, data):
+        return
+        #return super().feedbuffer(data)
+
     def _got_disconnected(self):
         self.connectstate = 'closed'
         self._send_rcpts({'connectstate': self.connectstate})

confluent_server/confluent/sshutil.py

@@ -89,11 +89,14 @@ def initialize_ca():
         ['ssh-keygen', '-C', comment, '-t', 'ed25519', '-f',
          '/etc/confluent/ssh/ca', '-N', get_passphrase()],
          preexec_fn=normalize_uid)
+    ouid = normalize_uid()
     try:
         os.makedirs('/var/lib/confluent/public/site/ssh/', mode=0o755)
     except OSError as e:
         if e.errno != 17:
             raise
+    finally:
+        os.seteuid(ouid)
     cafilename = '/var/lib/confluent/public/site/ssh/{0}.ca'.format(myname)
     shutil.copy('/etc/confluent/ssh/ca.pub', cafilename)
     #    newent = '@cert-authority * ' + capub.read()
@@ -167,16 +170,14 @@ def initialize_root_key(generate, automation=False):
             '-C', 'Confluent Automation by {}'.format(myname)],
             preexec_fn=normalize_uid)
         authorized = ['/etc/confluent/ssh/automation.pub']
+    ouid = normalize_uid()
     try:
         os.makedirs('/var/lib/confluent/public/site/ssh', mode=0o755)
-        neededuid = os.stat('/etc/confluent').st_uid
-        os.chown('/var/lib/confluent', neededuid, -1)
-        os.chown('/var/lib/confluent/public', neededuid, -1)
-        os.chown('/var/lib/confluent/public/site', neededuid, -1)
-        os.chown('/var/lib/confluent/public/site/ssh', neededuid, -1)
     except OSError as e:
         if e.errno != 17:
             raise
+    finally:
+        os.seteuid(ouid)
     neededuid = os.stat('/etc/confluent').st_uid
     if automation:
         suffix = 'automationpubkey'

confluent_server/confluent/syncfiles.py

@@ -128,15 +128,34 @@ class SyncList(object):
                         try:
                             uid = pwd.getpwnam(optval).pw_uid
                         except KeyError:
-                            uid = None
-                        optval = {'name': optval, 'id': uid}
+                            try:
+                                uid = int(optval)
+                                optval = None
+                            except ValueError:
+                                uid = None
+                        if optval:
+                            optval = {'name': optval}
+                        else:
+                            optval = {}
+                        if uid is not None:
+                            optval['id'] = uid
                     elif optname == 'group':
                         try:
                             gid = grp.getgrnam(optval).gr_gid
                         except KeyError:
-                            gid = None
-                        optval = {'name': optval, 'id': gid}
-                    entopts[optname] = optval
+                            try:
+                                gid = int(optval)
+                                optval = None
+                            except ValueError:
+                                gid = None
+                        if optval:
+                            optval = {'name': optval}
+                        else:
+                            optval = {}
+                        if gid is not None:
+                            optval['id'] = gid
+                    if optval:
+                        entopts[optname] = optval
             currmap[k] = v
             targ = v if v else k
             for f in targ.split():
@@ -205,6 +224,8 @@ def stage_ent(currmap, ent, targdir, appendexist=False):
     allfents = ent.split()
     for tmpent in allfents:
         fents = glob.glob(tmpent)
+        if not fents:
+            raise Exception('No matching files for "{}"'.format(tmpent))
         everyfent.extend(fents)
     if not everyfent:
         raise Exception('No matching files for "{}"'.format(ent))
@@ -233,8 +254,11 @@ def mkpathorlink(source, destination, appendexist=False):
         else:
             mkdirp(os.path.dirname(destination))
         if appendexist and os.path.exists(destination):
-            tmpnam = tempfile.mktemp()
-            shutil.copy(destination, tmpnam)
+            tmphdl, tmpnam = tempfile.mkstemp()
+            try:
+                shutil.copy(destination, tmpnam)
+            finally:
+                os.close(tmphdl)
             os.remove(destination)
             with open(destination, 'w') as realdest:
                 with open(tmpnam) as olddest:
@@ -243,6 +267,8 @@ def mkpathorlink(source, destination, appendexist=False):
                     realdest.write(sourcedata.read())
             os.remove(tmpnam)
         else:
+            if os.path.islink(destination):
+                os.remove(destination)
             os.symlink(source, destination)
 
 

confluent_server/confluent_server.spec.tmpl

@@ -14,9 +14,13 @@ Prefix: %{_prefix}
 BuildArch: noarch
 Requires: confluent_vtbufferd
 %if "%{dist}" == ".el8"
-Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
+Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
 %else
-Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python2-websocket-client python2-msgpack python-libarchive-c python-yaml
+%if "%{dist}" == ".el9"
+Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
+%else
+Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python2-websocket-client python2-msgpack python-libarchive-c python-yaml
+%endif
 %endif
 Vendor: Jarrod Johnson <jjohnson2@lenovo.com>
 Url: http://xcat.sf.net/
@@ -31,15 +35,23 @@ Server for console management and systems management aggregation
 %if "%{dist}" == ".el8"
 python3 setup.py build
 %else
+%if "%{dist}" == ".el9"
+python3 setup.py build
+%else
 python2 setup.py build
 %endif
+%endif
 
 %install
 %if "%{dist}" == ".el8"
 python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES.bare --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
 %else
+%if "%{dist}" == ".el9"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES.bare --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
+%else
 python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES.bare --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
 %endif
+%endif
 for file in $(grep confluent/__init__.py INSTALLED_FILES.bare); do
     rm $RPM_BUILD_ROOT/$file
 done

confluent_vtbufferd/builddeb

@@ -19,8 +19,8 @@ mv $PKGNAME $DPKGNAME-${VERSION}
 sed -i s/%%VERSION%%/$VERSION/ $DPKGNAME-${VERSION}/debian/changelog
 tar cvzf ${DPKGNAME}_${VERSION}.orig.tar.gz $DPKGNAME-${VERSION}
 cd $DPKGNAME-${VERSION}
-debuild -us -uc
+debuild --no-lintian -us -uc
 if [ ! -z "$1" ]; then
-    mv /tmp/confluent/$DPKGNAME_${VERSION}*.deb $1/
+    mv /tmp/confluent/${DPKGNAME}_${VERSION}*.deb $1/
 fi
 exit 0

confluent_vtbufferd/vtbufferd.c

@@ -178,22 +178,22 @@ int main(int argc, char* argv[]) {
         length = cmd & 536870911;
         cmd = cmd >> 29;
         if (cmd == SETNODE) {
-            currnode[length] = 0;
             cmd = fread(currnode, 1, length, stdin);
+            currnode[length] = 0;
             if (cmd < 0)
                 continue;
             currvt = set_termentbyname(currnode);
         } else if (cmd == WRITE) {
             if (currvt == NULL)
                 currvt = set_termentbyname("");
-            cmdbuf[length] = 0;
             cmd = fread(cmdbuf, 1, length, stdin);
+            cmdbuf[length] = 0;
             if (cmd < 0)
                 continue;
             tmt_write(currvt, cmdbuf, length);
         } else if (cmd == READBUFF) {
-            cmdbuf[length] = 0;
             cmd = fread(cmdbuf, 1, length, stdin);
+            cmdbuf[length] = 0;
             if (cmd < 0)
                 continue;
             outvt = get_termentbyname(cmdbuf);

genesis/97genesis/install-base

@@ -27,7 +27,7 @@ dracut_install opainfo
 dracut_install /usr/lib/udev/rules.d/10-dm.rules /usr/sbin/dmsetup /usr/lib/udev/rules.d/95-dm-notify.rules
 #dracut_install /usr/lib/opa-fm/bin/opafmd
 #dracut_install /usr/sbin/opensm /usr/libexec/opensm-launch
-dracut_install /usr/lib64/libibverbs/libhfi1verbs-rdmav25.so /etc/libibverbs.d/hfi1verbs.driver /etc/libibverbs.d/mlx4.driver /etc/libibverbs.d/mlx5.driver /usr/lib64/libibverbs/libmlx4-rdmav25.so /usr/lib64/libibverbs/libmlx5-rdmav25.so
+dracut_install /usr/lib64/libibverbs/libhfi1verbs-rdmav34.so /etc/libibverbs.d/hfi1verbs.driver /etc/libibverbs.d/mlx4.driver /etc/libibverbs.d/mlx5.driver /usr/lib64/libibverbs/libmlx4-rdmav34.so /usr/lib64/libibverbs/libmlx5-rdmav34.so
 inst /usr/lib/dracut/modules.d/40network/net-lib.sh /lib/net-lib.sh
 
 #dracut_install mstflint This would have mlx update support, but it incurs gigantic dependencies, libicudata

genesis/buildgenesis.sh

@@ -8,6 +8,7 @@ dracut --no-early-microcode --xz -N -m "genesis base" -f boot/initramfs/distribu
 cp -f /boot/vmlinuz-$(uname -r) boot/kernel
 cp /boot/efi/EFI/BOOT/BOOTX64.EFI boot/efi/boot
 cp /boot/efi/EFI/centos/grubx64.efi boot/efi/boot/grubx64.efi
+mkdir -p ~/rpmbuild/SOURCES/
 tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot
 rpmbuild -bb confluent-genesis.spec
 rm -rf /usr/lib/dracut/modules.d/97genesis

genesis/confluent-genesis.spec

@@ -1,6 +1,6 @@
 %define arch x86_64
-Version: 3.3.0
-Release:  2
+Version: 3.4.0
+Release:  1
 Name: confluent-genesis-%{arch}
 BuildArch: noarch
 Summary: Genesis servicing image for confluent

imgutil/confluent_imgutil.spec.tmpl

@@ -20,7 +20,7 @@ mkdir -p opt/confluent/lib/imgutil
 mkdir -p opt/confluent/bin
 mv imgutil opt/confluent/bin/
 chmod a+x opt/confluent/bin/imgutil
-mv ubuntu suse15 el8 opt/confluent/lib/imgutil/
+mv ubuntu suse15 el7 el8 opt/confluent/lib/imgutil/
 
 %install
 cp -a opt %{buildroot}/

imgutil/el7/dracut/install

@@ -0,0 +1,36 @@
+dracut_install mktemp
+dracut_install /lib64/libtss2-tcti-device.so.0
+dracut_install /lib64/libnsspem.so  /lib64/libsoftokn3.so  /lib64/libsqlite3.so.0
+
+dracut_install tpm2_create tpm2_pcrread tpm2_createpolicy tpm2_createprimary
+dracut_install tpm2_load tpm2_unseal tpm2_getcap tpm2_evictcontrol
+dracut_install tpm2_pcrextend tpm2_policypcr tpm2_flushcontext tpm2_startauthsession
+dracut_install curl openssl tar cpio gzip lsmod ethtool xz lsmod ethtool
+dracut_install modprobe touch echo cut wc bash uniq grep ip hostname
+dracut_install awk egrep dirname expr sort
+dracut_install ssh sshd reboot parted mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap
+dracut_install efibootmgr uuidgen
+dracut_install du df ssh-keygen scp clear dhclient
+dracut_install /lib64/libnss_dns.so.2
+dracut_install /usr/lib64/libnl-3.so.200
+dracut_install /etc/nsswitch.conf /etc/services /etc/protocols
+dracut_install chmod whoami head tail basename tr
+dracut_install /usr/sbin/arping /usr/sbin/dhclient-script ipcalc logger hostnamectl
+inst /bin/bash /bin/sh
+dracut_install /lib64/libfuse.so.2
+dracut_install chown chroot dd expr kill parted rsync sort blockdev findfs insmod lvm
+dracut_install /usr/lib/udev/rules.d/10-dm.rules /usr/sbin/dmsetup /usr/lib/udev/rules.d/95-dm-notify.rules
+dracut_install /usr/lib/udev/rules.d/60-net.rules /lib/udev/rename_device
+dracut_install losetup # multipart support
+
+#this would be nfs with lock, but not needed, go nolock
+#dracut_install mount.nfs rpcbind rpc.statd /etc/netconfig sm-notify
+#dracut_install mount.nfs  /etc/netconfig
+inst /usr/lib/dracut/modules.d/40network/net-lib.sh /lib/net-lib.sh
+
+
+
+# network mount, and disk imaging helpers can come from a second stage
+# this is narrowly focused on getting network up and fetching images
+# and those images may opt to do something with cloning or whatever
+

imgutil/el7/dracut/installkernel

@@ -0,0 +1,12 @@
+#!/bin/bash
+instmods nfsv3 nfs_acl nfsv4 dns_resolver lockd fscache sunrpc
+instmods e1000 e1000e igb sfc mlx5_ib mlx5_core mlx4_en cxgb3 cxgb4 tg3 bnx2 bnx2x bna ixgb ixgbe qlge mptsas mpt2sas mpt3sas megaraid_sas ahci xhci-hcd sd_mod pmcraid be2net vfat ext3 ext4 usb_storage scsi_wait_scan ipmi_si ipmi_devintf qlcnic xfs
+instmods nvme
+instmods cdc_ether
+instmods mptctl
+instmods mlx4_ib mlx5_ub ib_umad ib_ipoib
+instmods ice i40e hfi1 bnxt_en qed qede
+instmods dm-mod dm-log raid0 raid1 raid10 raid456 dm-raid dm-thin-pool dm-crypt dm-snapshot linear dm-era
+# nfs root and optionally gocryptfs
+instmods fuse overlay squashfs loop zram
+instmods aesni_intel drbg

imgutil/el7/pkglist

@@ -0,0 +1,19 @@
+yum
+hostname
+irqbalance
+less
+sssd-client
+NetworkManager
+nfs-utils
+numactl-libs
+passwd
+rootfiles
+sudo
+tuned
+yum
+initscripts
+xfsprogs
+e2fsprogs
+fuse-libs
+libnl3
+chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-x64.x86_64 grub2-efi-x64 attr

imgutil/el8/pkglist

@@ -17,4 +17,4 @@ xfsprogs
 e2fsprogs
 fuse-libs
 libnl3
-chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils clevis lvm2 efibootmgr shim-x64.x86_64 grub2-efi-x64 attr
+chrony kernel net-tools nfs-utils openssh-server rsync tar util-linux python3 tar dracut dracut-network ethtool parted openssl dhclient openssh-clients bash vim-minimal rpm iputils lvm2 efibootmgr shim-x64.x86_64 grub2-efi-x64 attr

imgutil/imgutil

@@ -340,7 +340,10 @@ def create_yumconf(sourcedir, addrepos):
     if '/' not in sourcedir:
         sourcedir = os.path.join('/var/lib/confluent/distributions', sourcedir)
     if os.path.exists(sourcedir + '/repodata'):
-        pass
+        yumconf.write('[genimage-topdir]\n')
+        yumconf.write('name=Local install repository\n')
+        yumconf.write('baseurl=file://{0}\n'.format(sourcedir))
+        yumconf.write('enabled=1\ngpgcheck=0\n\n')
     else:
         c = configparser.ConfigParser()
         c.read(sourcedir + '/.treeinfo')
@@ -548,7 +551,7 @@ class DebHandler(OsHandler):
 
 class ElHandler(OsHandler):
     def __init__(self, name, version, arch, args):
-        self.oscategory = 'el8'
+        self.oscategory = 'el{0}'.format(version.split('.')[0])
         self.yumargs = []
         super().__init__(name, version, arch, args)
 
@@ -876,7 +879,7 @@ def fingerprint_source_suse(files, sourcepath, args):
 
 def fingerprint_source_el(files, sourcepath, args):
     for filen in files:
-        if '-release-8' in filen:
+        if '-release-8' in filen or '-release-7' in filen:
             parts = filen.split('-')
             osname = '_'.join(parts[:-3])
             if osname == 'centos_linux':
@@ -901,19 +904,43 @@ def fingerprint_source(sourcepath, args):
     return oshandler
 
 def fingerprint_host_el(args, hostpath='/'):
+    if hostpath[0] != '/':
+         hostpath = os.path.join(os.getcwd(), hostpath)
     try:
         import rpm
+        ts = rpm.TransactionSet(hostpath)
+        rpms = ts.dbMatch('provides', 'system-release')
+        for inf in rpms:
+            if 'el8' not in inf.release and 'el7' not in inf.release:
+                continue
+            osname = inf.name
+            version = inf.version
+            release = inf.release
     except ImportError:
+        try:
+            rver = subprocess.check_output('rpm --root {0} -q --whatprovides system-release'.format(hostpath).split())
+            if not isinstance(rver, str):
+                rver = rver.decode('utf8')
+            for infline in subprocess.check_output('rpm -qi {0}'.format(rver).split()).decode('utf8').split('\n'):
+                if ':' not in infline:
+                    continue
+                k, v = infline.split(':', 1)
+                k = k.strip()
+                v = v.strip()
+                if k == 'Name':
+                    osname = v
+                elif k == 'Release':
+                    release = v
+                elif k == 'Version':
+                    version = v
+        except subprocess.SubprocessError:
+            return None
+    if 'el8' not in release and 'el7' not in release:
         return None
-    ts = rpm.TransactionSet(hostpath)
-    rpms = ts.dbMatch('provides', 'system-release')
-    for inf in rpms:
-        if 'el8' not in inf.release:
-            continue
-        osname = inf.name.replace('-release', '').replace('-', '_')
-        if osname == 'centos_linux':
-            osname = 'centos'
-        return ElHandler(osname, inf.version, os.uname().machine, args)
+    osname = osname.replace('-release', '').replace('-', '_')
+    if osname == 'centos_linux':
+        osname = 'centos'
+    return ElHandler(osname, version, os.uname().machine, args)
 
 
 def fingerprint_host_deb(args, hostpath='/'):

misc/fixsmmexpiry.py

@@ -31,7 +31,14 @@ if 'authResult>1' in rspdata:
      rspdata = rsp.read().decode('utf8')
 if 'renew_account' in rspdata:
      restorepwd = True
-     tokens = fromstring(rspdata)
+     if isinstance(rspdata, bytes):
+         if b'!entity' in rspdata.lower():
+             raise Exception('Unexpected material')
+     else:
+        if '!entity' in rspdata.lower():
+             raise Exception('Unexpected material')
+     # the troublesome entity tag is guarded above
+     tokens = fromstring(rspdata)  # nosec
      st2 = tokens.findall('st2')[0].text
      w.set_header('ST2', st2)
      w.request('POST', '/data/changepwd', 'oripwd={0}&newpwd={1}'.format(os.environ['SMMPASS'], tmppassword))
@@ -41,7 +48,14 @@ if 'renew_account' in rspdata:
      rsp = w.getresponse()
      rspdata = rsp.read().decode('utf8')
 if 'authResult>0' in rspdata:
-     tokens = fromstring(rspdata)
+     if isinstance(rspdata, bytes):
+        if b'!entity' in rspdata.lower():
+            raise Exception('Unexpected material')
+     else:
+        if '!entity' in rspdata.lower():
+             raise Exception('Unexpected material')
+     # the risky xml entity feature is filtered out above
+     tokens = fromstring(rspdata)  # nosec
      st2 = tokens.findall('st2')[0].text
      w.set_header('ST2', st2)
      rules = 'set=passwordDurationDays:0,passwordExpireWarningDays:0,passwordChangeInterval:0,passwordReuseCheckNum:0,passwordFailAllowdNum:0,passwordLockoutTimePeriod:0'

misc/setupssh.sh

@@ -0,0 +1,32 @@
+[ -f /lib/confluent/functions ] && . /lib/confluent/functions
+[ -f /etc/confluent/functions ] && . /etc/confluent/functions
+[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+nodename=$(grep ^NODENAME: /etc/confluent.info|awk '{print $NF}')
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    rm $certfile
+    confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
+done
+TMPDIR=$(mktemp -d)
+cd $TMPDIR
+confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs.tgz
+tar xf initramfs.tgz
+for ca in ssh/*.ca; do
+	LINE=$(cat $ca)
+	cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
+	grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	echo '@cert-authority *' $LINE >> /etc/ssh/ssh_known_hosts.new
+	mv /etc/ssh/ssh_known_hosts.new /etc/ssh/ssh_known_hosts
+done
+for pubkey in ssh/*.*pubkey; do
+	LINE=$(cat $pubkey)
+	cp -af /root/.ssh/authorized_keys /root/.ssh/authorized_keys.new
+	grep -v "$LINE" /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.new
+	echo "$LINE" >> /root/.ssh/authorized_keys.new
+	mv /root/.ssh/authorized_keys.new /root/.ssh/authorized_keys
+done
+confluentpython $confapiclient /confluent-api/self/nodelist | sed -e 's/^- //' > /etc/ssh/shosts.equiv
+cat /etc/ssh/shosts.equiv > /root/.shosts
+cd -
+rm -rf $TMPDIR