Fix function definition

Sourcing the functions was causing a hang.

confluent_client/bin/dir2img

@@ -1,19 +1,40 @@
-#!/bin/sh
+#!/usr/bin/python
 # This will take a given directory and make a 'big floppy image'
 # out of it, suitable for nodemedia upload.
 
-if [ -z "$1" -o -z "$2" ]; then
-        echo "Usage: $0 <directory> <imagefile>"
-        exit 1
-fi
-# Get the needed payload side
-SIZE=$(du -sB 512 $1|awk '{print $1}')
-ENTRIES=$(find $1 |wc -l)
-# Also, each file and directory has overhead, pad size by 32kb per file,
-# which should be overkill but other values proved to be inadequate
-# A deeper understanding of VFAT would probably allow for more precise value
-SIZE=$((SIZE + ENTRIES * 64))
-dd if=/dev/zero of=$2 bs=512 count=$SIZE
-# Make a big single sided floppy with many many tracks, saves on complex math
-mformat -i $2 -d 1 -t $SIZE -s 1 -h 1 ::
-mcopy -i $2 -s $1/* ::
+import glob
+import os
+import subprocess
+import sys
+
+def create_image(directory, image):
+    ents = 0
+    datasz = 512
+    for dir in os.walk(sys.argv[1]):
+        ents += 1
+        for filen in dir[2]:
+            ents += 1
+            filename = os.path.join(dir[0], filen)
+            currsz = os.path.getsize(filename)
+            # assuming up to 65k cluster
+            currsz = (currsz // 512 +1) * 512
+            datasz += currsz
+    datasz += ents * 32768
+    datasz = datasz // 512 + 1
+    with open(image, 'wb') as imgfile:
+        imgfile.seek(datasz * 512 - 1)
+        imgfile.write(b'\x00')
+    subprocess.check_call(['mformat', '-i', image, '-d', '1', '-t',
+                            str(datasz), '-s', '1','-h', '1', '::'])
+    cpycmd = ['mcopy', '-i', image, '-s']
+    cpycmd.extend(glob.glob('{0}/*'.format(directory)))
+    cpycmd.append('::')
+    subprocess.check_call(cpycmd)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        sys.stderr.write("Usage: {0} <directory> <imagefile>".format(
+            sys.argv[0]))
+        sys.exit(1)
+    create_image(sys.argv[1], sys.argv[2])

confluent_client/bin/nodedeploy

@@ -0,0 +1,67 @@
+#!/usr/bin/python3
+
+import argparse
+import os
+import sys
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def armonce(nr, cli):
+    nodes = set([])
+    for rsp in cli.read('/noderange/{0}/attributes/current'.format(nr)):
+        for node in rsp.get('databynode', {}):
+            nodeinfo = rsp['databynode'][node]
+            for attr in nodeinfo:
+                if attr == 'deployment.apiarmed':
+                    curr = nodeinfo[attr].get('value', '')
+                    if curr == 'continuous':
+                        nodes.add(node)
+    noderange = nr
+    if nodes:
+        noderange += ',-({0})'.format(','.join(nodes))
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(noderange),
+                          {'deployment.apiarmed': 'once'}):
+        pass
+
+def setpending(nr, profile, cli):
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(nr),
+                          {'deployment.pendingprofile': profile}):
+        pass
+
+
+def main(args):
+    ap = argparse.ArgumentParser(description='Deploy OS to nodes')
+    ap.add_argument('-n', '--network', help='Initiate deployment over PXE', action='store_true')
+    ap.add_argument('-m', '--maxnodes', help='Specifiy a maximum nodes to be deployed')
+    ap.add_argument('noderange', help='Set of nodes to deploy')
+    ap.add_argument('profile', help='Profile name to deploy')
+    args = ap.parse_args(args)
+    if not args.network:
+        sys.stderr.write('Currently only network (-n) deployment is supported\n')
+        return 1
+    c = client.Command()
+    c.stop_if_noderange_over(args.noderange, args.maxnodes)
+    armonce(args.noderange, c)
+    setpending(args.noderange, args.profile, c)
+    errnodes = set([])
+    rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                     bootmode='uefi',
+                                     persistent=False,
+                                     errnodes=errnodes)
+    if errnodes:
+        sys.stderr.write(
+            'Unable to set boot device for following nodes: {0}\n'.format(
+                ','.join(errnodes)))
+        return 1
+    rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
+    return rc
+
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

confluent_client/bin/nodeshell

@@ -46,10 +46,12 @@ def run():
                          help='Number of commands to run at a time')
     argparser.add_option('-n', '--nonodeprefix', action='store_true',
                          help='Do not prefix output with node names')
+    argparser.add_option('-p', '--port', type='int', default=0,
+                         help='Specify a custom port for ssh')
     argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to run remote ssh command to, '
-                          'prompting if over the threshold')   
+                          'prompting if over the threshold')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -59,7 +61,7 @@ def run():
         sys.exit(1)
     client.check_globbing(args[0])
     concurrentprocs = options.count
-    c = client.Command()    
+    c = client.Command()
     cmdstr = " ".join(args[1:])
 
     currprocs = 0
@@ -79,7 +81,10 @@ def run():
             cmd = ex[node]['value']
             if not isinstance(cmd, str) and not isinstance(cmd, bytes):
                 cmd = cmd.encode('utf-8')
-            cmdv = ['ssh', node, cmd]
+            if options.port:
+                cmdv = ['ssh', '-p', '{0}'.format(options.port), node, cmd]
+            else:
+                cmdv = ['ssh', node, cmd]
             if currprocs < concurrentprocs:
                 currprocs += 1
                 run_cmdv(node, cmdv, all, pipedesc)

confluent_client/confluent_env.sh

@@ -19,6 +19,7 @@ alias nodebmcreset='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export
 alias nodeboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeboot'
 alias nodeconfig='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeconfig'
 alias nodeconsole='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeconsole'
+alias nodedeploy='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodedeploy'
 alias nodedefine='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodedefine'
 alias nodeeventlog='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeeventlog'
 alias nodefirmware='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodefirmware'
@@ -137,6 +138,34 @@ _confluent_nodefirmware_completion()
     fi
 }
 
+_confluent_osimage_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 2 ]; then
+        COMPREPLY=($(compgen -W "initialize import" -- ${COMP_WORDS[COMP_CWORD]}))
+        return
+    elif [ ${CMPARGS[1]} == 'initialize' ]; then
+        COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]}))
+    elif [ ${CMPARGS[1]} == 'import' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+}
+
+_confluent_nodedeploy_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}"))
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
 _confluent_nodeshell_completion()
 {
     _confluent_get_args
@@ -266,6 +295,8 @@ complete -F _confluent_nr_completion nodeconfig
 complete -F _confluent_nn_completion nodeconsole
 complete -F _confluent_nr_completion nodeeventlog
 complete -F _confluent_nodefirmware_completion nodefirmware
+complete -F _confluent_nodedeploy_completion nodedeploy
+complete -F _confluent_osimage_completion osimage
 complete -F _confluent_ng_completion nodegroupattrib
 complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth

confluent_client/doc/man/nodediscover.ronn

@@ -23,6 +23,9 @@ data may be filtered by various parameters, as denoted in the options below.
 **nodediscover assign** performs manual discovery, assigning an entry to a node
 identity or, using `-i`, using a csv file to assign nodes all at once.  For
 example, a spreadsheet of serial numbers to desired node names could be used.
+Note that if you see that the host is unreachable, it may be due to the IP
+address on the endpoint having changed since last detected. In such a case, it
+may help to **clear** and try **assign** again.
 
 **nodediscover rescan** requests the server to do an active sweep for new
 devices.  Generally every effort is made to passively detect devices as they

confluent_osdeploy/buildrpm

@@ -0,0 +1,11 @@
+VERSION=`git describe|cut -d- -f 1`
+NUMCOMMITS=`git describe|cut -d- -f 2`
+if [ "$NUMCOMMITS" != "$VERSION"  ]; then
+    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+fi
+sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec
+cd ..
+tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy
+mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
+cd -
+rpmbuild -ba confluent_osdeploy.spec

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -0,0 +1,47 @@
+Name: confluent_osdeploy-x86_64
+Version: #VERSION#
+Release: 1
+Summary: OS Deployment support for confluent
+
+License: Apache2
+URL: http://hpc.lenovo.com/
+Source0: confluent_osdeploy.tar.xz
+BuildArch: noarch
+Requires: confluent_ipxe mtools
+BuildRoot: /tmp
+
+%description
+This contains support utilities for enabling deployment of x86_64 architecture systems
+
+
+%define debug_package %{nil}
+
+%prep
+%setup -n confluent_osdeploy
+
+%build
+mkdir -p opt/confluent/bin
+cd utils
+make all
+cp copernicus clortho autocons ../opt/confluent/bin
+cd ..
+for os in el8 suse15 ubuntu20.04; do
+    mkdir ${os}out
+    cd ${os}out
+    cp -a ../opt .
+    cp -a ../${os}/initramfs/* .
+    find . | cpio -H newc -o > ../addons.cpio
+    mv ../addons.cpio .
+    cd ..
+done
+
+%install
+for os in el8 suse15 ubuntu20.04; do
+    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
+    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
+    cp ${os}out/addons.cpio %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
+    cp -a $os/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
+done
+
+%files
+/opt/confluent/lib/osdeploy

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+echo -n "" >> /tmp/net.ifaces
+cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    if [ -n "$autocons" ]; then
+        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+        autocons=${autocons%,*}
+	echo $autocons > /tmp/01-autocons.devnode
+	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
+        echo "Initializing auto detected console when installer starts" > $autocons
+    fi
+fi
+. /lib/anaconda-lib.sh
+wait_for_kickstart

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+[ -e /tmp/confluent.initq ] && return 0
+echo -n "" > /tmp/confluent.initq
+cd /sys/class/net
+for currif in *; do
+    ip link set $currif up
+done
+cd -
+while ! grep MANAGER /tmp/confluent.info >& /dev/null; do
+    /opt/confluent/bin/copernicus -t > /tmp/confluent.info
+done
+read ifidx <<EOF
+$(grep ^MANAGER /tmp/confluent.info|grep fe80|sed -e s/.*%//)
+EOF
+read mgr << EOF
+$(grep ^MANAGER /tmp/confluent.info|grep fe80|awk '{print $2}')
+EOF
+mgridx=${mgr#*%}
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+echo $ifname > /tmp/net.ifaces
+nodename=$(grep ^NODENAME /tmp/confluent.info|awk '{print $2}')
+#TODO: blkid --label <whatever> to find mounted api
+
+if [ -z "$apikey" ]; then
+    apikey=$(/opt/confluent/bin/clortho $nodename $mgr)
+fi
+oum=$(umask)
+umask 0077
+echo $apikey > /etc/confluent.apikey
+umask $oum
+mgr="[$mgr]"
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg > /tmp/confluent.deploycfg
+
+dnsdomain=$(grep ^dnsdomain: /tmp/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+hostname=$nodename
+if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+    hostname=$hostname.$dnsdomain
+fi
+mgr=$(grep ^deploy_server: /tmp/confluent.deploycfg)
+mgr=${mgr#deploy_server: }
+profilename=$(grep ^profile: /tmp/confluent.deploycfg)
+profilename=${profilename#profile: }
+proto=$(grep ^protocol: /tmp/confluent.deploycfg)
+proto=${proto#protocol: }
+textconsole=$(grep ^textconsole: /tmp/confluent.deploycfg)
+textconsole=${textconsole#textconsole: }
+if [ $textconsole = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+	autocons=$(cat /tmp/01-autocons.devnode)
+	if [ ! -z "$autocons" ]; then
+	    echo Auto-configuring installed system to use text console
+	    echo Auto-configuring installed system to use text console > $autocons
+	    cp /tmp/01-autocons.conf /etc/cmdline.d/
+	else
+	    echo "Unable to automatically detect requested text console"
+	fi
+fi
+
+echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf
+echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf
+kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
+root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
+export kickstart
+export root
+autoconfigmethod=$(grep ipv4_method /tmp/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+else
+    v4addr=$(grep ^ipv4_address: /tmp/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /tmp/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: /tmp/confluent.deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
+fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /tmp/confluent.deploycfg
+

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+while [ -h $BUNDLENAME ]; do
+    BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
+done
+
+cat /etc/pki/tls/certs/ca-bundle.crt > $BUNDLENAME
+mkdir -p /sysroot/etc/confluent/
+cp -a /tls /sysroot/etc/confluent
+sed -i 's/install::/install:*:/' /sysroot/etc/shadow
+sed -i 's/root::/root:*:/' /sysroot/etc/shadow
+mkdir -p /sysroot/root/.ssh
+chmod 700 /sysroot/root/.ssh
+cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+chmod 600 /sysroot/root/.ssh/authorized_keys
+mkdir -p /sysroot/etc/ssh/
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+cp /etc/confluent.apikey /sysroot/etc/
+cp /etc/confluent.apikey /sysroot/etc/confluent/
+cp /tmp/confluent.deploycfg /tmp/confluent.info /sysroot/etc/
+cp /tmp/confluent.deploycfg /tmp/confluent.info /sysroot/etc/confluent

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
+ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+

confluent_osdeploy/el8/profiles/default/kickstart

@@ -0,0 +1,65 @@
+# In this OS profile, data is largely filled in during the %pre
+# phase, rather than the kickstart actually having the content.
+# None of the files shall be replaced during an upgrade in
+# /var/lib/confluent/public/os/<profile>, so customization should
+# be done by modifying files in /var/lib/confluent/public/os/<profile>
+
+# /tmp/rootpw will provide a 'rootpw' line, either locking password if not configured
+# or the crypted form.
+%include /tmp/rootpw
+
+# timezone is fetched from confluent server, which provides the
+# timezone that the management server itself is in by default.
+%include /tmp/timezone
+
+# similar to timezone, confluent is asked to provide the
+# deployment servers language info and replicate that
+# to the deployment target.
+%include /tmp/langinfo
+
+# The default partition scheme is applied to a single drive, using
+# the getinstalldisk script to make a best guess as to the most
+# appropriate device. See pre.sh and getinstalldisk to customize
+# the automatic behavior, or comment out/delete the 
+# following line and provide your own manual partition plan
+# instead
+%include /tmp/partitioning
+
+reboot
+
+%packages
+@^minimal-environment
+chrony
+rsync
+python3
+%end
+
+%pre
+profile=$(grep ^profile: /etc/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/preinst.sh
+. /tmp/preinst.sh
+%end
+
+%post --nochroot
+mkdir -p /mnt/sysimage/etc/confluent
+profile=$(grep ^profile: /etc/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+
+# Hook firstboot.sh
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
+chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+%end
+
+%post
+systemctl enable firstboot
+chgrp ssh_keys /etc/ssh/ssh*key
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+%end

confluent_osdeploy/el8/profiles/default/profile.yaml

@@ -0,0 +1,2 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet

confluent_osdeploy/el8/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/etc/confluent/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+curl --capath /etc/confluent/tls -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -0,0 +1,14 @@
+run_remote() {
+    cd $(mktemp -d)
+    curl -f https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    chmod +x $1
+    ./$1
+    cd -
+}
+
+run_remote_python() {
+    cd $(mktemp -d)
+    curl -f https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    /usr/libexec/platform-python $1
+    cd -
+}

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# need to copy over ssh key info
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+export mgr profile nodename
+. /etc/confluent/functions
+
+# This script will execute in the installed system, but using the installer kernel prior to reboot.
+# This is an appropriate place to run post install activities that do not require the actual installed
+# kernel to run. For example adding drivers that would be needed for first boot to run cleanly.
+# If, for example, there is a post script that has a dependency on a driver or filesystem that
+# cannot work until booting into the installer, use firstboot.sh instead
+
+# run_remote will download and execute from /var/lib/confluent/public/<os>/scripts/ directory
+# run_remote_python will use the appropriate python interpreter path to run the specified script
+
+# Add content as below:
+
+# run_remote example.sh
+# run_remote_python example.py

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# This runs prior to the installer beginning. This is used to rewrite the 
+# scripted install file, merging data from confluennt and identifying
+# the most appropriate install source.
+
+# If you want to use a more custom partition plan, the easiest
+# method is to edit the kicktstart file and comment out or
+# delete %include /tmp/partitioning
+
+nodename=$(grep ^NODENAME /etc/confluent.info|awk '{print $2}')
+locale=$(grep ^locale: /etc/confluent.deploycfg)
+locale=${locale#locale: }
+keymap=$(grep ^keymap: /etc/confluent.deploycfg)
+keymap=${keymap#keymap: }
+echo lang $locale > /tmp/langinfo
+echo keyboard --vckeymap=$keymap >> /tmp/langinfo
+tz=$(grep ^timezone: /etc/confluent.deploycfg)
+tz=${tz#timezone: }
+echo timezone $tz --utc > /tmp/timezone
+rootpw=$(grep ^rootpassword /etc/confluent.deploycfg | awk '{print $2}')
+if [ "$rootpw" = null ]; then
+    echo "rootpw --lock" > /tmp/rootpw
+else
+    echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
+fi
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent.apikey)" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config.anaconda
+done
+/usr/sbin/sshd -f /etc/ssh/sshd_config.anaconda
+if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
+    consoledev=$(cat /run/install/cmdline.d/01-autocons.conf | sed -e 's!console=!/dev/!' -e 's/,.*//')
+    tmux a <> $consoledev >&0 2>&1 &
+fi
+export mgr profile nodename
+curl -f https://$mgr/confluent-public/os/$profile/scripts/functions > /tmp/functions
+. /tmp/functions
+run_remote_python getinstalldisk
+if [ -e /tmp/installdisk ]; then
+    echo clearpart --all --initlabel >> /tmp/partitioning
+    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
+    echo autopart --nohome >> /tmp/partitioning
+fi

confluent_osdeploy/el8/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# This script runs after install is complete, but inside the installer 
+# environment. This is useful for carrying work done in pre/during the
+# installer into the installed environment.
+
+# It is almost certainly more useful to use post.sh or firstboot.sh
+# for customization, which will run in a more normal mechanism
+
+nodename=$(grep ^NODENAME /etc/confluent.info|awk '{print $2}')
+export mgr profile nodename
+cp -a /etc/confluent /mnt/sysimage/etc
+cp /tmp/functions /mnt/sysimage/etc/confluent/
+. /tmp/functions
+
+# Preserve the ssh setup work done for the installer
+# by copying into the target system and setting up
+# host based authentication
+run_remote setupssh.sh

confluent_osdeploy/el8/profiles/default/scripts/setupssh.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+grep HostCert /etc/ssh/sshd_config.anaconda >> /mnt/sysimage/etc/ssh/sshd_config
+echo HostbasedAuthentication yes >> /mnt/sysimage/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /mnt/sysimage/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /mnt/sysimage/etc/ssh/sshd_config
+sshconf=/etc/ssh/ssh_config
+if [ -d /mnt/sysimage/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/mnt/sysimage/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+
+cp /etc/ssh/ssh_host_* /mnt/sysimage/etc/ssh/
+mkdir /mnt/sysimage/root/.ssh/
+chmod 700 /mnt/sysimage/root/.ssh/
+cp /root/.ssh/authorized_keys /mnt/sysimage/root/.ssh/
+chmod 600 /mnt/sysimage/root/.ssh/authorized_keys
+cp /etc/ssh/ssh_known_hosts /mnt/sysimage/etc/ssh/
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent.apikey)" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
+cp /tmp/allnodes /mnt/sysimage/etc/ssh/shosts.equiv
+cp /tmp/allnodes /mnt/sysimage/root/.shosts

confluent_osdeploy/suse15/initramfs/etc/linuxrc.d/01-confluent

@@ -0,0 +1 @@
+Install: exec:/opt/confluent/bin/suseagent

confluent_osdeploy/suse15/initramfs/opt/confluent/bin/suseagent

@@ -0,0 +1,106 @@
+#!/bin/bash
+echo "Installing certificates"
+echo '<authorized_keys xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns" config:type="list">' > /tmp/rootkeys.xml
+for pub in /ssh/*.rootpubkey; do
+    echo '<listentry>'$(cat $pub)'</listentry>' >> /tmp/rootkeys.xml
+done
+echo '</authorized_keys>' >> /tmp/rootkeys.xml
+/usr/bin/cp /tls/*.0 /var/lib/ca-certificates/openssl/
+/usr/bin/cp /tls/*.0 /etc/ssl/certs/
+echo "LineMode: 1" > /etc/linuxrc.d/01-confluent
+autocons=""
+if ! grep console /proc/cmdline > /dev/null; then
+	autocons=$(/opt/confluent/bin/autocons)
+	if [ ! -z "$autocons" ]; then
+		echo "Serial console detected from firmmware: $autocons" > ${autocons%,*}
+	fi
+fi
+cd /sys/class/net
+for nic in *; do
+	ip link set $nic up
+done
+echo -n "Discovering confluent..."
+/opt/confluent/bin/copernicus -t > /tmp/confluent.info
+while ! grep MANAGER: /tmp/confluent.info > /dev/null; do
+	/opt/confluent/bin/copernicus -t > /tmp/confluent.info
+done
+nodename=$(grep ^NODENAME: /tmp/confluent.info | head -n 1 | sed -e 's/NODENAME: //')
+echo "done ($nodename)"
+echo "Hostname: $nodename" >> /etc/linuxrc.d/01-confluent
+mgr=$(grep ^MANAGER: /tmp/confluent.info | head -n 1 | sed -e 's/MANAGER: //')
+echo -n "Acquiring configuration from $mgr..."
+bootifidx=${mgr#*%}
+for nic in *; do
+	if [ "$(cat $nic/ifindex)" = "$bootifidx" ]; then
+		bootif=$nic
+	fi
+done
+cd -
+echo "NetDevice: $bootif" >> /etc/linuxrc.d/01-confluent
+/opt/confluent/bin/clortho $nodename $mgr > /tmp/confluent.apikey
+mgr="[$mgr]"
+curl -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /tmp/confluent.apikey)" https://$mgr/confluent-api/self/deploycfg > /tmp/confluent.deploycfg
+dnsdomain=$(grep ^dnsdomain: /tmp/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+	echo "Domain: $dnsdomain" >> /etc/linuxrc.d/01-confluent
+fi
+textconsole=$(grep ^textconsole: /tmp/confluent.deploycfg)
+textconsole=${textconsole#textconsole: }
+if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null && [ ! -z "$autocons" ]; then
+	echo "Serial console autodetected and enabled, will be available upon install completion" > ${autocons%,*}
+	echo "The installer will run in text mode on the graphics console. When ssh is available," > ${autocons%,*}
+	echo "install progress may be checked by using ssh to access and run the screendump command" > ${autocons%,*}
+	echo "Install time serial console would require the profile to have console=${autocons#/dev/}" > ${autocons%,*}
+	echo "specified in the kernel command line arguments" > ${autocons%,*}
+	sed -e s'/$/ 'console=${autocons#*/dev/}/ /proc/cmdline > /etc/fakecmdline
+	mount -o bind /etc/fakecmdline /proc/cmdline
+	echo "ConsoleDevice: ${autocons%,*}" >> /etc/linuxrc.d/01-confluent
+fi
+
+tz=$(grep timezone: /tmp/confluent.deploycfg | awk '{print $2}')
+echo "<timezone>${tz}</timezone>" > /tmp/timezone
+autoconfigmethod=$(grep ipv4_method /tmp/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+if [ "$autoconfigmethod" = "dhcp" ]; then
+	echo "DHCP: 1" >> /etc/linuxrc.d/01-confluent
+else
+	v4addr=$(grep ^ipv4_address: /tmp/confluent.deploycfg)
+	v4addr=${v4addr#ipv4_address: }
+	v4gw=$(grep ^ipv4_gateway: /tmp/confluent.deploycfg)
+	v4gw=${v4gw#ipv4_gateway: }
+	v4nm=$(grep ipv4_netmask: /tmp/confluent.deploycfg)
+	v4nm=${v4nm#ipv4_netmask: }
+	echo "HostIP: $v4addr" >> /etc/linuxrc.d/01-confluent
+	echo "Netmask: $v4nm" >> /etc/linuxrc.d/01-confluent
+	if [ "$v4gw" != "null" ]; then
+		echo "Gateway: $v4gw" >> /etc/linuxrc.d/01-confluent
+	fi
+	nameserversec=0
+	while read -r entry; do
+		if [ $nameserversec = 1 ]; then
+			if [[ $entry == "-"* ]]; then
+				echo Nameserver: ${entry#- } >> /etc/linuxrc.d/01-confluent
+				continue
+			fi
+		fi
+		nameserversec=0
+		if [ ${entry%:*} = "nameservers" ]; then
+			nameserversec=1
+			continue
+		fi
+	done < /tmp/confluent.deploycfg
+fi
+echo done
+mgr=$(grep ^deploy_server: /tmp/confluent.deploycfg)
+mgr=${mgr#deploy_server: }
+profilename=$(grep ^profile: /tmp/confluent.deploycfg)
+profilename=${profilename#profile: }
+proto=$(grep ^protocol: /tmp/confluent.deploycfg)
+proto=${proto#protocol: }
+
+echo "<media_url>${proto}://${mgr}/confluent-public/os/${profilename}/product</media_url>" > /tmp/pkgurl
+
+echo "AutoYaST: $proto://$mgr/confluent-public/os/$profilename/autoyast" >> /etc/linuxrc.d/01-confluent
+echo "Install: $proto://$mgr/confluent-public/os/$profilename/distribution/1" >> /etc/linuxrc.d/01-confluent
+exec /init

confluent_osdeploy/suse15/profiles/hpc/autoyast

@@ -0,0 +1,135 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile SYSTEM "/usr/share/YaST2/include/autoinstall/profile.dtd">
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns" xmlns:xi="http://www.w3.org/2001/XInclude">
+    <general>
+      <self_update config:type="boolean">false</self_update>
+      <clock>
+        <hwclock>UTC</hwclock>
+        <xi:include href="file:///tmp/timezone"/>
+      </clock>
+      <mode>
+	      <confirm config:type="boolean">false</confirm>
+      </mode>
+    </general>
+    %%IFSLE%%
+    <add-on>
+        <add_on_products config:type="list">
+            <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-hpc</product><product_dir>/Module-HPC</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-server-applications</product><product_dir>/Module-Server-Applications</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>sle-module-containers</product><product_dir>/Module-Containers</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>SLE_HPC</product><product_dir>/Product-HPC</product_dir></listentry>
+	    <listentry><xi:include href="file:///tmp/pkgurl"/><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
+        </add_on_products>
+</add-on>
+    %%ENDIFSLE%%
+    <software>
+       %%IFSLE%%
+       <products config:type="list">
+          <product>SLE_HPC</product>
+      </products>
+       %%ENDIFSLE%%
+      <patterns config:type="list">
+        <pattern>base</pattern>
+      </patterns>
+      <packages config:type="list">
+        <package>openssl</package>
+        <package>chrony</package>
+        <package>rsync</package>
+        <package>screen</package>
+        <package>vim</package>
+        <package>binutils</package>
+        <package>pciutils</package>
+        <package>usbutils</package>
+        <package>nfs-client</package>
+        <package>ethtool</package>
+      </packages>
+    </software>
+  <partitioning config:type="list">
+	  <drive>
+		  <device>%%INSTDISK%%</device>
+		  <initialize config:type="boolean">true</initialize>
+		  <use>all</use>
+                  <partitions config:type="list">
+                         <partition>
+                                 <filesystem config:type="symbol">xfs</filesystem>
+                                 <mount>/</mount>
+                                 <size>max</size>
+                         </partition>
+                         <partition>
+                                 <mount>swap</mount>
+                                 <size>auto</size>
+                         </partition>
+                         <partition>
+                                 <mount>/boot</mount>
+                                 <size>500M</size>
+                         </partition>
+                 </partitions>
+          </drive>
+ </partitioning>
+    <users config:type="list">
+      <user>
+        <username>root</username>
+        <user_password>%%ROOTPASSWORD%%</user_password>
+        <encrypted config:type="boolean">true</encrypted>
+	<xi:include href="file:///tmp/rootkeys.xml"/>
+      </user>
+    </users>
+    <networking>
+	    <dns>
+		    <hostname>%%NODENAME%%</hostname>
+            </dns>
+	    <keep_install_network config:type="boolean">true</keep_install_network>
+    </networking>
+    <services-manager>
+	    <services>
+		    <enable config:type="list">
+			    <service>sshd</service>
+			</enable>
+		</services>
+	</services-manager>
+    <scripts>
+         <pre-scripts config:type="list">
+        <script>
+          <filename>preinstall.sh</filename>
+          <interpreter>shell</interpreter>
+          <source>
+<![CDATA[
+#!/bin/sh
+mgr=$(grep ^deploy_server /tmp/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /tmp/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /tmp/confluent.deploycfg |awk '{print $2}')
+curl $proto://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh
+]]>
+          </source>
+        </script>
+      </pre-scripts>
+         <chroot-scripts config:type="list">
+        <script>
+          <filename>chroot.sh</filename>
+          <interpreter>shell</interpreter>
+          <source>
+<![CDATA[
+#!/bin/sh
+mgr=$(grep ^deploy_server /tmp/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /tmp/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /tmp/confluent.deploycfg |awk '{print $2}')
+curl $proto://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/post.sh
+. /tmp/post.sh
+curl $proto://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/etc/confluent/firstboot.sh
+chmod +x /mnt/etc/confluent/firstboot.sh
+]]>
+          </source>
+        </script>
+      </chroot-scripts>
+      <init-scripts config:type="list">
+        <script>
+          <filename>post.sh</filename>
+          <interpreter>shell</interpreter>
+	  <location>file:///etc/confluent/firstboot.sh</location>
+        </script>
+     </init-scripts> 
+
+    </scripts>
+</profile>

confluent_osdeploy/suse15/profiles/hpc/initprofile.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+discnum=$(basename $1)
+if [ "$discnum" != 1 ]; then exit 0; fi
+if [ -e $2/boot/kernel ]; then exit 0; fi
+if ls $1/Product-* >& /dev/null; then
+	ln -s $1 $2/product
+else
+	ln -s ${1%1}2 $2/product
+fi
+sed -i 's/sle 15/SUSE Linux Enterprise 15/; s/opensuse_leap/openSUSE Leap/' $2/profile.yaml
+ln -s $1/boot/x86_64/loader/linux $2/boot/kernel && \
+ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
+profile=$(basename $2)
+if [[ $profile =~ ^sle.* ]]; then
+	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
+else
+	sed -i 's/%%IFSLE%%/<!--/;s/%%ENDIFSLE%%/-->/' $2/autoyast
+fi

confluent_osdeploy/suse15/profiles/hpc/profile.yaml

@@ -0,0 +1,2 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# This script runs at the end of the final boot, updating status
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+curl --capath /etc/confluent/tls -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -f -X POST -d "status: complete" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# This script runs when install is finished, but while the installer
+# is still running, with the to-be-booted system mounted in /mnt
+
+# Carry over install-time ssh material into installed system
+mkdir -p /mnt/root/.ssh/
+chmod 700 /mnt/root/.ssh/
+cp /root/.ssh/authorized_keys /mnt/root/.ssh/
+chmd 600 /mnt/root/.ssh/authorized_keys
+cp /etc/ssh/*key* /mnt/etc/ssh/
+for i in /etc/ssh/*-cert.pub; do
+    echo HostCertificate $i >> /mnt/etc/ssh/sshd_config
+done
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /mnt/etc/ssh/ssh_known_hosts
+done
+# Enable ~/.shosts, for the sake of root user, who is forbidden from using shosts.equiv
+echo IgnoreRhosts no >> /mnt/etc/ssh/sshd_config
+echo HostbasedAuthentication yes >> /mnt/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /mnt/etc/ssh/sshd_config
+echo Host '*' >> /mnt/etc/ssh/ssh_config
+echo "    HostbasedAuthentication yes" >> /mnt/etc/ssh/ssh_config
+echo "    EnableSSHKeysign yes" >> /mnt/etc/ssh/ssh_config
+# Limit the attempts of using host key. This prevents client from using 3 or 4
+# authentication attempts through host based attempts
+echo "    HostbasedKeyTypes *ed25519*" >> /mnt/etc/ssh/ssh_config
+
+# In SUSE platform, setuid for ssh-keysign is required for host based,
+# and also must be opted into.
+echo /usr/lib/ssh/ssh-keysign root:root 4711 >> /mnt/etc/permissions.local
+chmod 4711 /mnt/usr/lib/ssh/ssh-keysign
+
+# Download list of nodes from confluent, and put it into shosts.equiv (for most users) and .shosts (for root)
+nodename=$(grep ^NODENAME /tmp/confluent.info|awk '{print $2}')
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /tmp/confluent.apikey)" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
+cp /tmp/allnodes /mnt/root/.shosts
+cp /tmp/allnodes /mnt/etc/ssh/shosts.equiv
+
+# carry over deployment configuration and api key for OS install action
+mkdir -p /mnt/etc/confluent
+chmod 700 /mnt/etc/confluent
+chmod 600 /tmp/confluent.*
+cp /tmp/confluent.* /mnt/etc/confluent/
+cp -a /tls /mnt/etc/confluent/
+cp -a /tls/* /mnt/var/lib/ca-certificates/openssl
+cp -a /tls/* /mnt/var/lib/ca-certificates/pem
+cp -a /tls/*.pem /mnt/etc/pki/trust/anchors
+

confluent_osdeploy/suse15/profiles/hpc/scripts/pre.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# This script runs before the installer executes, and sets up ssh during install as well
+# as rewriting the autoyast file with any substitutions prior to it being evaluated for real
+
+nodename=$(grep ^NODENAME /tmp/confluent.info|awk '{print $2}')
+rootpw=$(grep rootpassword: /tmp/confluent.deploycfg|sed -e 's/^rootpassword: //')
+if [ "$rootpw" = "null" ]; then
+    rootpw="!"
+fi
+
+mkdir ~/.ssh
+cat /ssh/*.rootpubkey > ~/.ssh/authorized_keys
+ssh-keygen -A
+for i in  /etc/ssh/ssh_host*key.pub; do
+    certname=${i/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /tmp/confluent.apikey)" -d @$i https://$mgr/confluent-api/self/sshcert > $certname
+    echo HostKey ${i%.pub} >> /etc/ssh/sshd_config
+    echo HostCertificate $certname >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+curl -f https://$mgr/confluent-public/os/$profile/scripts/getinstalldisk > /tmp/getinstalldisk
+python3 /tmp/getinstalldisk
+sed -e s!%%INSTDISK%%!/dev/$(cat /tmp/installdisk)! -e s!%%NODENAME%%!$nodename! -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml

confluent_osdeploy/ubuntu20.04/initramfs/conf/conf.d/confluent

@@ -0,0 +1,8 @@
+if ! grep console= /proc/cmdline > /dev/null; then
+    /opt/confluent/bin/autocons > /custom-installation/autocons.info
+    cons=$(cat /custom-installation/autocons.info)
+    if [ ! -z "$cons" ]; then
+        echo "Auto-detected serial console: $cons" > ${cons%,*}
+    fi
+fi
+echo /scripts/init-premount/confluent >> /scripts/init-premount/ORDER

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/hooks/casper-bottom.sh

@@ -0,0 +1,67 @@
+cd /root
+cat /tls/*.0 > /root/etc/ssl/certs/ca-certificates.crt
+mkdir -p /root/custom-installation/ssh
+mkdir -p /root/custom-installation/tls
+cp /ssh/* /root/custom-installation/ssh
+cp /tls/* /root/custom-installation/tls
+NODENAME=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}')
+MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}')
+oum=$(umask)
+umask 077
+chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+MGR=[$MGR]
+deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
+umask $oum
+nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+nic=$(ip link |grep ^$nic:|awk '{print $2}')
+DEVICE=${nic%:}
+ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+. /scripts/functions
+if [ "$ipv4m" = "dhcp" ]; then
+    IP=dhcp
+    configure_networking
+elif [ "$ipv4m" = "static" ]; then
+    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    dnsdomain=$(grep ^dnsdomain: $deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    if [ "$dnsdomain" = "null" ]; then dnsdomain=""; fi
+    dns=$(grep -A1 ^nameservers: $deploycfg|head -n 2|tail -n 1|sed -e 's/^- //'|sed -e "s/''//")
+    {
+        echo "DEVICE='$DEVICE'"
+        echo "PROTO='none'"
+        echo "IPV4PROTO='none'"
+        echo "IPV4ADDR='$v4addr'"
+        echo "IPV4NETMASK='$v4nm'"
+        echo "IPV4BROADCAST='$v4nm'"
+        echo "IPV4GATEWAY='$v4gw'"
+        echo "IPV4DNS1='$dns'"
+        echo "HOSTNAME='$NODENAME'"
+        echo "DNSDOMAIN='$dnsdomain'"
+        echo "DOMAINSEARCH='$dnsdomain'"
+    } > "/run/net-$DEVICE.conf"
+    configure_networking
+else
+    IP=off
+fi
+ipv4s=$(grep ^deploy_server $deploycfg|awk '{print $2}')
+osprofile=$(cat /custom-installation/confluent/osprofile)
+fcmdline="$(cat /custom-installation/confluent/cmdline.orig) autoinstall ds=nocloud-net;s=https://${ipv4s}/confluent-public/os/${osprofile}/autoinstall/"
+if [ -f /custom-installation/autocons.info ]; then
+    cons=$(cat /custom-installation/autocons.info)
+fi
+if [ ! -z "$cons" ]; then
+    echo "Installation will proceed on graphics console, autoconsole not supported during autoinstall phase" > ${cons%,*}
+    echo "Progress can be checked by using ssh to access and running the screendump command" > ${cons%,*}
+    #fcmdline="$fcmdline console=${cons#/dev/}"
+fi
+echo $fcmdline > /custom-installation/confluent/fakecmdline
+/scripts/casper-bottom/58server_network

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/post.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+deploycfg=/custom-installation/confluent/confluent.deploycfg
+mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}')
+profile=$(grep ^profile: $deploycfg|awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/post.sh
+. /tmp/post.sh

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/pre.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+deploycfg=/custom-installation/confluent/confluent.deploycfg
+mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}')
+profile=$(grep ^profile: $deploycfg|awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -0,0 +1,30 @@
+cd /sys/class/net
+for nic in *; do
+    ip link set $nic up
+done
+mkdir -p /custom-installation
+cp -a /opt/confluent /custom-installation
+touch /custom-installation/confluent/confluent.info
+while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+done
+MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
+cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
+if [ -f /custom-installation/autocons.info ]; then
+    cons=$(cat /custom-installation/autocons.info)
+fi
+if [ ! -z "$cons" ]; then
+    echo "Preparing to deploy $osprofile from $MGR" > ${cons%,*}
+fi
+echo "Preparing to deploy $osprofile from $MGR"
+echo $osprofile > /custom-installation/confluent/osprofile
+echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
+fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso"
+if [ ! -z "$cons" ]; then
+    fcmdline="$fcmdline console=${cons#/dev/}"
+fi
+echo $fcmdline > /custom-installation/confluent/fakecmdline
+mount -o bind /custom-installation/confluent/fakecmdline /proc/cmdline
+
+

confluent_osdeploy/ubuntu20.04/profiles/default/autoinstall/user-data

@@ -0,0 +1,17 @@
+#cloud-config
+autoinstall:
+  version: 1
+  early-commands:
+    - /custom-installation/pre.sh
+  late-commands:
+    - /custom-installation/post.sh
+  ssh:
+    install-server: true
+  storage:
+    layout:
+      name: lvm
+      match:
+        path: "%%INSTALLDISK%%"
+  user-data:
+    runcmd:
+      - /etc/confluent/firstboot.sh

confluent_osdeploy/ubuntu20.04/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \
+ln -s $1/casper/vmlinuz $2/boot/kernel && \
+ln -s $1/casper/initrd $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/* $2/boot/efi/boot
+

confluent_osdeploy/ubuntu20.04/profiles/default/profile.yaml

@@ -0,0 +1,2 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet osprofile=%%PROFILE%%

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+echo "Confluent first boot is running"
+cp -a /etc/confluent/ssh/* /etc/ssh/
+systemctl restart sshd
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
+	echo root:$rootpw | chpasswd -e
+fi
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+touch /etc/cloud/cloud-init.disabled
+curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -X POST -d "status: complete" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+cp -a /root/.ssh /target/root/
+mkdir -p /target/etc/confluent/ssh/sshd_config.d/
+chmod 700 /target/etc/confluent
+cp /custom-installation/confluent/* /target/etc/confluent/
+cp -a /custom-installation/tls /target/etc/confluent/
+chmod go-rwx /etc/confluent/*
+for i in /custom-installation/ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts
+done
+
+cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/
+cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/
+sshconf=/target/etc/ssh/ssh_config
+if [ -d /target/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/target/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
+chmod +x /target/etc/confluent/firstboot.sh
+cp /tmp/allnodes /target/root/.shosts
+cp /tmp/allnodes /target/etc/ssh/shosts.equiv
+textcons=$(grep ^textconsole: /target/etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+    cons=""
+    if [ -f /custom-installation/autocons.info ]; then
+        cons=$(cat /custom-installation/autocons.info)
+    fi
+    if [ ! -z "$cons" ]; then
+        sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /target/etc/default/grub
+	mount -o bind /dev /target/dev
+	mount -o bind /proc /target/proc
+	mount -o bind /sys /target/sys
+	chroot /target update-grub
+	umount /target/sys /target/dev /target/proc
+    fi
+fi
+

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/pre.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+cat /custom-installation/ssh/*.rootpubkey > /root/.ssh/authorized_keys
+nodename=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /custom-installation/confluent/confluent.apikey)
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    keyfile=${pubkey%.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
+    echo HostKey $keyfile >> /etc/ssh/sshd_config.d/confluent.conf
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config.d/confluent.conf
+done
+echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
+echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
+echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
+systemctl restart sshd
+curl -f X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
+curl -f https://$mgr/confluent-public/os/$profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
+python3 /custom-installation/getinstalldisk
+sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml

confluent_osdeploy/utils/Makefile

@@ -0,0 +1,16 @@
+CC := gcc
+CFLAGS := -Os
+TARGETS := copernicus autocons
+
+all: $(TARGETS) clortho
+
+clortho: clortho.c
+	$(CC) $(CFLAGS) -o $@ $^ -lcrypt
+	strip -s $@
+
+$(TARGETS): % : %.c
+	$(CC) $(CFLAGS) -o $@ $^
+	strip -s $@
+
+clean:
+	rm $(TARGETS) clortho

confluent_osdeploy/utils/autocons.c

@@ -25,6 +25,7 @@ int main(int argc, char* argv[]) {
     int currspeed;
     speed_t cspeed;
     char buff[128];
+    char* offset;
     uint64_t address;
     spcr = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
     if (spcr < 0) {
@@ -38,6 +39,7 @@ int main(int argc, char* argv[]) {
     if (buff[40] != 1) exit(0); //IO only
     address = *(uint64_t *)(buff + 44);
     currspeed = buff[58];
+    offset = buff + 10;
     if (address == COM1) {
         strncpy(buff, "/dev/ttyS0", 128);
     } else if (address == COM2) {
@@ -51,14 +53,16 @@ int main(int argc, char* argv[]) {
     }
     if (currspeed == SPEED9600) {
         cspeed = B9600;
+        strcpy(offset, ",9600");
     } else if (currspeed == SPEED19200) {
         cspeed = B19200;
+        strcpy(offset, ",19200");
     } else if (currspeed == SPEED57600) {
         cspeed = B57600;
+        strcpy(offset, ",57600");
     } else if (currspeed == SPEED115200) {
         cspeed = B115200;
-    } else if (currspeed == SPEED115200) {
-        cspeed = 0;
+        strcpy(offset, ",115200");
     } else {
         exit(0);
     }

confluent_osdeploy/utils/clortho.c

@@ -0,0 +1,130 @@
+/* Copyright 2019 Lenovo */
+#include <arpa/inet.h>
+#include <crypt.h>
+#include <net/if.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MAXPACKET 1024
+
+static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./";
+
+unsigned char* genpasswd(int len) {
+        unsigned char * passwd;
+        int urandom, ret;
+        passwd = calloc(len + 1, sizeof(char));
+        urandom = open("/dev/urandom", O_RDONLY);
+        ret = read(urandom, passwd, len);
+        close(urandom);
+        for (urandom = 0; urandom < len; urandom++) {
+                passwd[urandom] = cryptalpha[passwd[urandom] >> 2];
+        }
+        return passwd;
+
+}
+
+
+int main(int argc, char* argv[]) {
+        int sock, ret;
+        char slen;
+        unsigned char currlen, currtype;
+        unsigned char* passwd;
+        unsigned char* cryptedpass;
+        unsigned char* macaddr;
+        struct timeval timeout;
+        struct addrinfo hints;
+        struct addrinfo *addrs;
+        struct addrinfo *curr;
+        struct sockaddr_in net4bind;
+        struct sockaddr_in6 net6bind;
+        unsigned char buffer[MAXPACKET];
+        memset(&hints, 0, sizeof(struct addrinfo));
+        memset(&net4bind, 0, sizeof(struct sockaddr_in));
+        memset(&net6bind, 0, sizeof(struct sockaddr_in6));
+        memset(&buffer, 0, MAXPACKET);
+        memset(&timeout, 0, sizeof(struct timeval));
+        timeout.tv_sec = 10;
+        net4bind.sin_port = htons(302);
+        net6bind.sin6_port = htons(302);
+        net6bind.sin6_family = AF_INET6;
+        hints.ai_socktype = SOCK_STREAM;
+        hints.ai_protocol = IPPROTO_TCP;
+
+        passwd = genpasswd(32);
+        memset(buffer, 0, MAXPACKET);
+        strncpy(buffer, "$5$", 3);
+        cryptedpass = genpasswd(8);
+        strncpy(buffer + 3, cryptedpass, 8);
+        free(cryptedpass);
+        cryptedpass = crypt(passwd, buffer);
+        if (argc < 3) {
+            fprintf(stderr, "Missing node name and manager\n");
+            exit(1);
+        }
+        sock = getaddrinfo(argv[2], "13001", &hints, &addrs);
+        if (sock != 0) {
+                fprintf(stderr, "Error trying to resolve %s\n", argv[2]);
+                exit(1);
+        }
+        for (curr = addrs; curr != NULL; curr = curr->ai_next) {
+                sock = socket(curr->ai_family, curr->ai_socktype, curr->ai_protocol);
+                if (sock < 0) continue;
+                setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
+                if (curr->ai_family == AF_INET) {
+                        bind(sock, (struct sockaddr*)&net4bind, sizeof(struct sockaddr_in));
+                } else if (curr->ai_family == AF_INET6) {
+                        bind(sock, (struct sockaddr*)&net6bind, sizeof(struct sockaddr_in6));
+                } else {
+                        continue;
+                }
+                if (connect(sock, curr->ai_addr, curr->ai_addrlen) == 0) break;
+        }
+        if (curr == NULL) {
+                fprintf(stderr, "Unable to reach %s\n", argv[2]);
+                exit(1);
+        }
+        setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
+        freeaddrinfo(addrs);
+        ret = read(sock, buffer, 8);
+        if (memcmp(buffer, "\xc2\xd1-\xa8\x80\xd8j\xba", 8) != 0) {
+                fprintf(stderr, "Unrecognized server\n");
+                exit(1);
+        }
+        slen = strlen(argv[1]) & 0xff;
+        dprintf(sock, "\x01%c%s", slen, argv[1]);
+        ret = write(sock, "\x00\x00", 2);
+        memset(buffer, 0, MAXPACKET);
+        ret = read(sock, buffer, 2);
+        while (buffer[0] != 255) {
+            currtype = buffer[0];
+            currlen = buffer[1];
+            memset(buffer, 0, MAXPACKET);
+            if (currlen) {
+                ret = read(sock, buffer, currlen);  // Max is 255, well under MAX_PACKET
+            }
+            if (currtype == 2) {
+                dprintf(sock, "\x03%c", currlen);
+                ret = write(sock, buffer, currlen);
+                slen = strlen(cryptedpass) & 0xff;
+                dprintf(sock, "\x04%c%s", slen, cryptedpass);
+                ret = write(sock, "\x00\x00", 2);
+            } else if (currtype == 5) {
+                printf("%s", passwd);
+                printf("\n");
+                exit(0);
+            }
+            buffer[0] = 255;
+            ret = read(sock, buffer, 2);
+        }
+        fprintf(stderr, "Password was not accepted\n");
+        exit(1);
+}

confluent_osdeploy/utils/copernicus.c

@@ -48,8 +48,7 @@ int add_macs(char* destination, int maxsize) {
         } else if (lla->sll_hatype == ARPHRD_ETHER) {
             snprintf(macaddr, 32, "/mac=%02x:%02x:%02x:%02x:%02x:%02x",
                 lla->sll_addr[0], lla->sll_addr[1], lla->sll_addr[2],
-                lla->sll_addr[3], lla->sll_addr[4], lla->sll_addr[5],
-                lla->sll_addr[6]
+                lla->sll_addr[3], lla->sll_addr[4], lla->sll_addr[5]
             );
         } else {
             continue;
@@ -68,10 +67,16 @@ int main(int argc, char* argv[]) {
     struct sockaddr_in6 addr, dst;
     struct sockaddr_in addr4, dst4;
     char msg[1024];
+    char *nodenameidx;
+    char nodename[1024];
+    char lastnodename[1024];
     char lastmsg[1024];
+    char last6msg[1024];
     int ifidx, offset;
     fd_set rfds;
     struct timeval tv;
+    int settime = 0;
+    int setusec = 500000;
     socklen_t dstsize, dst4size;
     dstsize = sizeof(dst);
     dst4size = sizeof(dst4);
@@ -80,6 +85,10 @@ int main(int argc, char* argv[]) {
     memset(&addr, 0, sizeof(addr));
     memset(&dst, 0, sizeof(dst));
     memset(&dst4, 0, sizeof(dst4));
+    memset(nodename, 0, 1024);
+    memset(lastnodename, 0, 1024);
+    memset(lastmsg, 0, 1024);
+    memset(last6msg, 0, 1024);
     addr.sin6_family = AF_INET6;
     addr.sin6_addr = in6addr_any;
     addr.sin6_port = htons(190);
@@ -94,10 +103,6 @@ int main(int argc, char* argv[]) {
     inet_pton(AF_INET, "239.255.255.250", &dst4.sin_addr);
     strncpy(msg,  "M-SEARCH * HTTP/1.1\r\nST: urn:xcat.org:service:confluent:", 1024);
     offset = strnlen(msg, 1024);
-    if (argc > 1) {
-        snprintf(msg + offset, 1024 - offset, "/node=%s", argv[1]);
-        offset = strnlen(msg, 1024);
-    }
     add_uuid(msg + offset, 1024 - offset);
     offset = strnlen(msg, 1024);
     add_macs(msg + offset, 1024 - offset);
@@ -134,30 +139,100 @@ int main(int argc, char* argv[]) {
     FD_ZERO(&rfds);
     FD_SET(n4, &rfds);
     FD_SET(ns, &rfds);
-    tv.tv_sec = 10;
-    tv.tv_usec = 0;
+    tv.tv_sec = 2;
+    tv.tv_usec = 500000;
     ifidx = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
     while (ifidx) {
         if (ifidx == -1) perror("Unable to select");
         if (ifidx) {
             if (FD_ISSET(n4, &rfds)) {
-                recvfrom(n4, msg, 1024, 0, (struct sockaddr *)&dst4, &dst4size);
+                memset(msg, 0, 1024);
+                /* Deny packet access to the last 24 bytes to assure null */
+                recvfrom(n4, msg, 1000, 0, (struct sockaddr *)&dst4, &dst4size);
+                if  (nodenameidx = strstr(msg, "NODENAME: ")) {
+                        nodenameidx += 10;
+                        strncpy(nodename, nodenameidx, 1024);
+                        nodenameidx = strstr(nodename, "\r");
+                        if (nodenameidx) { nodenameidx[0] = 0; }
+                        if (strncmp(lastnodename, nodename, 1024) != 0) {
+                            printf("NODENAME: %s\n", nodename);
+                            strncpy(lastnodename, nodename, 1024);
+                        }
+                }
+                if (nodenameidx = strstr(msg, "CURRTIME: ")) {
+                    nodenameidx += 10;
+                    strncpy(nodename, nodenameidx, 1024);
+                    if (nodenameidx = strstr(nodename, "\r")) {
+                        nodenameidx[0] = 0;
+                    }
+                    settime = strtol(nodename, NULL, 10);
+                }
+                if (nodenameidx = strstr(msg, "CURRMSECS: ")) {
+                    nodenameidx += 10;
+                    strncpy(nodename, nodenameidx, 1024);
+                    if (nodenameidx = strstr(nodename, "\r")) {
+                        nodenameidx[0] = 0;
+                    }
+                    setusec = strtol(nodename, NULL, 10) * 1000;
+                }
+                memset(msg, 0, 1024);
                 inet_ntop(dst4.sin_family, &dst4.sin_addr, msg, dst4size);
                 /* Take measure from printing out the same ip twice in a row */
                 if (strncmp(lastmsg, msg, 1024) != 0) {
-                    printf("%s\n", msg);
+                    sendto(n4, "PING", 4, 0, (const struct sockaddr *)&dst4, dst4size);
+                    printf("MANAGER: %s\n", msg);
                     strncpy(lastmsg, msg, 1024);
                 }
             }
             if (FD_ISSET(ns, &rfds)) {
-                recvfrom(ns, msg, 1024, 0, (struct sockaddr *)&dst, &dstsize);
+                memset(msg, 0, 1024);
+                /* Deny packet access to the last 24 bytes to assure null */
+                recvfrom(ns, msg, 1000, 0, (struct sockaddr *)&dst, &dstsize);
+                if  (nodenameidx = strstr(msg, "NODENAME: ")) {
+                        nodenameidx += 10;
+                        strncpy(nodename, nodenameidx, 1024);
+                        nodenameidx = strstr(nodename, "\r");
+                        if (nodenameidx) { nodenameidx[0] = 0; }
+                        if (strncmp(lastnodename, nodename, 1024) != 0) {
+                            printf("NODENAME: %s\n", nodename);
+                            strncpy(lastnodename, nodename, 1024);
+                        }
+                }
+                if (nodenameidx = strstr(msg, "CURRTIME: ")) {
+                    nodenameidx += 10;
+                    strncpy(nodename, nodenameidx, 1024);
+                    if (nodenameidx = strstr(nodename, "\r")) {
+                        nodenameidx[0] = 0;
+                    }
+                    settime = strtol(nodename, NULL, 10);
+                }
+                if (nodenameidx = strstr(msg, "CURRMSECS: ")) {
+                    nodenameidx += 10;
+                    strncpy(nodename, nodenameidx, 1024);
+                    if (nodenameidx = strstr(nodename, "\r")) {
+                        nodenameidx[0] = 0;
+                    }
+                    setusec = strtol(nodename, NULL, 10) * 1000;
+                }
+                memset(msg, 0, 1024);
                 inet_ntop(dst.sin6_family, &dst.sin6_addr, msg, dstsize);
-                if (strncmp(lastmsg, msg, 1024) != 0) {
-                    printf("%s\n", msg);
-                    strncpy(lastmsg, msg, 1024);
+                if (strncmp(last6msg, msg, 1024) != 0) {
+                    sendto(ns, "PING", 4, 0, (const struct sockaddr *)&dst, dstsize);
+                    printf("MANAGER: %s", msg);
+                    if (strncmp(msg, "fe80::", 6) == 0) {
+                        printf("%%%u", dst.sin6_scope_id);
+                    }
+                    printf("\n");
+                    strncpy(last6msg, msg, 1024);
                 }
             }
         }
+        if (settime && argc > 1 && strcmp(argv[1], "-t") == 0) {
+            tv.tv_sec = settime;
+            tv.tv_usec = setusec;
+            settimeofday(&tv, NULL);
+            settime = 0;
+        }
         tv.tv_sec = 0;
         tv.tv_usec = 500000;
         FD_SET(n4, &rfds);

confluent_server/bin/confluentcertutil.py

@@ -1,55 +0,0 @@
-from os.path import exists
-import shutil
-import socket
-import subprocess
-import tempfile
-
-def get_openssl_conf_location():
-    if exists('/etc/pki/tls/openssl.cnf'):
-        return '/etc/pki/tls/openssl.cnf'
-    elif exists('/etc/ssl/openssl.cnf'):
-        return '/etc/ssl/openssl.cnf'
-    else:
-        raise Exception("Cannot find openssl config file")
-
-def get_ip_addresses():
-    lines = subprocess.check_output('ip addr'.split(' '))
-    for line in lines.split('\n'):
-        if line.startswith('    inet6 '):
-            line = line.replace('    inet6 ', '').split('/')[0]
-            if line.startswith('fe80::'):
-                continue
-            if line == '::1':
-                continue
-        elif line.startswith('    inet '):
-            line = line.replace('    inet ', '').split('/')[0]
-            if line == '127.0.0.1':
-                continue
-            if line.startswith('169.254.'):
-                continue
-        else:
-            continue
-        yield line
-
-def create_certificate():
-    shortname = socket.gethostname().split('.')[0]
-    longname = socket.getfqdn()
-    subprocess.check_call(
-        'openssl ecparam -name secp384r1 -genkey -out privkey.pem'.split(' '))
-    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
-    san.append('DNS:{0}'.format(shortname))
-    san.append('DNS:{0}'.format(longname))
-    san = ','.join(san)
-    sslcfg = get_openssl_conf_location()
-    tmpconfig = tempfile.mktemp()
-    shutil.copy2(sslcfg, tmpconfig)
-    with open(tmpconfig, 'a') as cfgfile:
-        cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san))
-    subprocess.check_call(
-        'openssl req -new -x509 -key privkey.pem -days 7300 -out cert.pem '
-        '-subj /CN={0} -extensions SAN '
-        '-config {1}'.format(longname, tmpconfig).split(' ')
-    )
-
-if __name__ == '__main__':
-    create_certificate()

confluent_server/bin/confluentdbutil

@@ -62,8 +62,11 @@ if args[0] == 'restore':
     if pid is not None:
         print("Confluent is running, must shut down to restore db")
         sys.exit(1)
+    password = options.password
+    if options.interactivepassword:
+        password = getpass.getpass('Enter password to restore backup: ')
     try:
-        cfm.restore_db_from_directory(dumpdir, options.password)
+        cfm.restore_db_from_directory(dumpdir, password)
     except Exception as e:
         print(str(e))
         sys.exit(1)
@@ -86,7 +89,7 @@ elif args[0] == 'dump':
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)
-    cfm.dump_db_to_directory(dumpdir, options.password, options.redact,
+    cfm.dump_db_to_directory(dumpdir, password, options.redact,
                              options.skipkeys)
 
 

confluent_server/bin/osimage

@@ -0,0 +1,186 @@
+#!/usr/bin/python2
+
+import argparse
+import eventlet.green.subprocess as subprocess
+import glob
+import os
+import os.path
+import shutil
+import sys
+import time
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sshutil as sshutil
+import confluent.certutil as certutil
+try:
+    input = raw_input
+except NameError:
+    pass
+
+fnamechars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.^'
+def main(args):
+    ap = argparse.ArgumentParser(description='Manage OS deployment resources')
+    sp = ap.add_subparsers(dest='command')
+    wiz = sp.add_parser('initialize', help='Do OS deployment preparation')
+    wiz.add_argument('-u', help='Pull in root user key for node deployment', action='store_true')
+    wiz.add_argument('-s', help='Set up SSH CA for managing node to node ssh and known hosts', action='store_true')
+    wiz.add_argument('-k', help='Update local global known hosts file with confluent CA', action='store_true')
+    wiz.add_argument('-t', help='Generate new TLS key for HTTPS operation and register with confluent repository', action='store_true')
+    wiz.add_argument('-p', help='Copy in TFTP contents required for PXE support', action='store_true')
+    wiz.add_argument('-i', help='Interactively prompt for behaviors', action='store_true')
+    osip = sp.add_parser('import', help='Import an OS image from an ISO image')
+    osip.add_argument('imagefile', help='File to use for source of importing')
+    cmdset = ap.parse_args()
+    if cmdset.command == 'import':
+        return osimport(cmdset.imagefile)
+    if cmdset.command == 'initialize':
+        return initialize(cmdset)
+    ap.print_help()
+
+def install_tftp_content():
+    tftplocation = None
+    candidates = ('/var/lib/tftpboot', '/srv/tftpboot', '/srv/tftp')
+    for cand in candidates:
+        if os.path.isdir(cand):
+            tftplocation = cand
+            break
+    if not tftplocation:
+        raise Exception('Unable to detect an installed tftp location')
+    tftplocation = '{0}/confluent/x86_64'.format(tftplocation)
+    try:
+        os.makedirs(tftplocation)
+    except OSError as e:
+        if e.errno == 17:
+            raise
+    shutil.copy('/opt/confluent/lib/ipxe/ipxe.efi', tftplocation)
+    shutil.copy('/opt/confluent/lib/ipxe/ipxe.kkpxe', tftplocation)
+
+
+def initialize(cmdset):
+    if os.getuid() != 0:
+        sys.stderr.write('This command must run as root user\n')
+        sys.exit(1)
+    if cmdset.i:
+        didsomething = True
+        sys.stdout.write('Add root user key to be authorized to log into nodes (-u)? (y/n): ')
+        sys.stdout.flush()
+        cmdset.u = input().strip().lower().startswith('y')
+        sys.stdout.write('Set up an SSH authority to help manage known_hosts and node to node ssh for all users (-s)? (y/n): ')
+        cmdset.s = input().strip().lower().startswith('y')
+        sys.stdout.write('Update global known hosts on this server to trust local CA certificates (-k)? (y/n): ')
+        cmdset.k = input().strip().lower().startswith('y')
+        sys.stdout.write('Update tftp directory with binaries to support PXE (-p) (y/n): ')
+        cmdset.p = input().strip().lower().startswith('y')
+        sys.stdout.write('Generate new TLS certificates for HTTP, replacing any existing certificate (-t)? (y/n): ')
+        cmdset.t = input().strip().lower().startswith('y')
+        if not cmdset.t:
+            print(
+                'In order to use your own certificate authority, make sure '
+                'to put the certificate authority into '
+                '/var/lib/confluent/public/site/tls/ directory as a .pem file '
+                'as well as named (hash).0 where (hash) is the hash of the '
+                'subject.')
+    else:
+        didsomething = False
+    if cmdset.u:
+        if not glob.glob('/root/.ssh/*.pub'):
+            didsomething = True
+            sys.stderr.write('No user keys for root detected, it is recommended '
+                            'to run ssh-keygen -t ed25519 to generate a user '
+                            'key. For optimal security, a passphrase should be '
+                            'used. ssh-agent may be used to make use of a '
+                            'passphrase protected ssh key easier.\n')
+            sys.exit(1)
+        sshutil.initialize_root_key(False)
+    if cmdset.t:
+        didsomething = True
+        certutil.create_certificate()
+        print('New HTTPS certificates generated, restart the web server')
+    if cmdset.s:
+        didsomething = True
+        sshutil.initialize_ca()
+
+    if not didsomething:
+        sys.stderr.write('Nothing was done, use initialize -i for '
+                        'interactive mode, or see initialize -h for more options\n')
+        sys.exit(1)
+    tmpname = '/var/lib/confluent/public/site/initramfs.cpio.'
+    for x in bytearray(os.urandom(22)):
+        tmpname += fnamechars[x >> 2]
+    topack = []
+    opath = os.getcwd()
+    os.chdir('/var/lib/confluent/public/site')
+    topack.append('ssh/')
+    for currd, _, files in os.walk('ssh'):
+        for fname in files:
+            topack.append(os.path.join(currd, fname))
+    topack.append('tls/')
+    for currd, _, files in os.walk('tls'):
+        for fname in files:
+            topack.append(os.path.join(currd, fname))
+    with open(tmpname, 'wb') as initramfs:
+        packit = subprocess.Popen(['cpio', '-H', 'newc', '-o'],
+                                  stdout=initramfs, stdin=subprocess.PIPE)
+        for packfile in topack:
+            if not isinstance(packfile, bytes):
+                packfile = packfile.encode('utf8')
+            packit.stdin.write(packfile)
+            packit.stdin.write(b'\n')
+        packit.stdin.close()
+        res = packit.wait()
+        if res:
+            sys.stderr.write('Error occurred while packing site initramfs')
+            sys.exit(1)
+    os.chdir(opath)
+    os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio')
+    if cmdset.k:
+        with open('/etc/ssh/ssh_known_hosts', 'a+b') as skh:
+            for cafile in glob.glob('/var/lib/confluent/public/site/ssh/*.ca'):
+                cacert = open(cafile, 'rb').read()
+                cacert = b'@cert-authority * ' + cacert
+                skh.write(cacert)
+    if cmdset.p:
+        install_tftp_content()
+    # ok, also need to think on how to handle getinstalldisk
+    if not os.path.exists('/etc/confluent/srvcert.pem'):
+        subprocess.check_call(['collective', 'gencert'])
+    # TODO: check selinux and segetbool for httpd_can_network_connect 
+    # tftp-server available and enabled?
+    # httpd available and enabled?
+
+
+def osimport(imagefile):
+    c = client.Command()
+    imagefile = os.path.abspath(imagefile)
+    importing = False
+    shortname = None
+    for rsp in c.create('/deployment/importing/', {'filename': imagefile}):
+        if 'target' in rsp:
+            importing = True
+            shortname = rsp['name']
+            print('Importing from {0} to {1}'.format(imagefile, rsp['target']))
+        else:
+            print(repr(rsp))
+    while importing:
+        for rsp in c.read('/deployment/importing/{0}'.format(shortname)):
+            if 'progress' in rsp:
+                sys.stdout.write('{0}: {1:.2f}%    \r'.format(rsp['phase'],
+                                                          rsp['progress']))
+                if rsp['phase'] == 'complete':
+                    importing = False
+                    sys.stdout.write('\n')
+                    for profile in rsp['profiles']:
+                        print('Deployment profile created: {0}'.format(profile))
+                sys.stdout.flush()
+            else:
+                print(repr(rsp))
+            time.sleep(0.5)
+    list(c.delete('/deployment/importing/{0}'.format(shortname)))
+
+if __name__ == '__main__':
+    main(sys.argv)

confluent_server/builddeb

@@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyte, python3-pyghmi, python3-paramiko/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
@@ -44,6 +44,9 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
         echo 'confluent_client confluent-client' >> debian/pydist-overrides
     fi
 fi
+if ! grep wheezy /etc/os-release; then
+    sed -i 's/^Package: python3-/Package: /' debian/control
+fi
 head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
 echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules

confluent_server/confluent/certutil.py

@@ -0,0 +1,130 @@
+import os
+import confluent.collective.manager as collective
+from os.path import exists
+import shutil
+import socket
+import eventlet.green.subprocess as subprocess
+import tempfile
+
+def get_openssl_conf_location():
+    if exists('/etc/pki/tls/openssl.cnf'):
+        return '/etc/pki/tls/openssl.cnf'
+    elif exists('/etc/ssl/openssl.cnf'):
+        return '/etc/ssl/openssl.cnf'
+    else:
+        raise Exception("Cannot find openssl config file")
+
+def get_ip_addresses():
+    lines = subprocess.check_output('ip addr'.split(' '))
+    if not isinstance(lines, str):
+        lines = lines.decode('utf8')
+    for line in lines.split('\n'):
+        if line.startswith('    inet6 '):
+            line = line.replace('    inet6 ', '').split('/')[0]
+            if line == '::1':
+                continue
+        elif line.startswith('    inet '):
+            line = line.replace('    inet ', '').split('/')[0]
+            if line == '127.0.0.1':
+                continue
+            if line.startswith('169.254.'):
+                continue
+        else:
+            continue
+        yield line
+
+def check_apache_config(path):
+    keypath = None
+    certpath = None
+    with open(path, 'r') as openf:
+        webconf = openf.read()
+    for line in webconf.split('\n'):
+        line = line.strip()
+        line = line.split('#')[0]
+        if line.startswith('SSLCertificateFile'):
+            _, certpath = line.split(None, 1)
+        if line.startswith('SSLCertificateKeyFile'):
+            _, keypath = line.split(None, 1)
+    return keypath, certpath
+
+def get_certificate_paths():
+    keypath = None
+    certpath = None
+    if os.path.exists('/etc/httpd/conf.d/ssl.conf'): # redhat way
+        keypath, certpath = check_apache_config('/etc/httpd/conf.d/ssl.conf')
+    if not keypath and os.path.exists('/etc/apache2'): # suse way
+        for currpath, _, files in os.walk('/etc/apache2'):
+            for fname in files:
+                if fname.endswith('.template'):
+                    continue
+                kploc = check_apache_config(os.path.join(currpath,
+                                                                       fname))
+                if keypath and kploc[0]:
+                    return None, None # Ambiguous...
+                if kploc[0]:
+                    keypath, certpath = kploc
+
+    return keypath, certpath
+
+def create_certificate(keyout=None, certout=None):
+    if not keyout:
+        keyout, certout = get_certificate_paths()
+    if not keyout:
+        raise Exception('Unable to locate TLS certificate path automatically')
+    shortname = socket.gethostname().split('.')[0]
+    longname = socket.getfqdn()
+    subprocess.check_call(
+        ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
+         keyout])
+    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
+    # It is incorrect to put IP addresses as DNS type.  However
+    # there exists non-compliant clients that fail with them as IP
+    san.extend(['DNS:{0}'.format(x) for x in get_ip_addresses()])
+    san.append('DNS:{0}'.format(shortname))
+    san.append('DNS:{0}'.format(longname))
+    san = ','.join(san)
+    sslcfg = get_openssl_conf_location()
+    tmpconfig = tempfile.mktemp()
+    shutil.copy2(sslcfg, tmpconfig)
+    try:
+        with open(tmpconfig, 'a') as cfgfile:
+            cfgfile.write('\n[SAN]i\nbasicConstraints = CA:true\nsubjectAltName={0}'.format(san))
+        subprocess.check_call([
+            'openssl', 'req', '-new', '-x509', '-key', keyout, '-days',
+            '7300', '-out', certout, '-subj', '/CN={0}'.format(longname),
+            '-extensions', 'SAN', '-config', tmpconfig
+        ])
+    finally:
+        os.remove(tmpconfig)
+    # Could restart the webserver now?
+    fname = '/var/lib/confluent/public/site/tls/{0}.pem'.format(
+        collective.get_myname())
+    try:
+        os.makedirs(os.path.dirname(fname))
+    except OSError as e:
+        if e.errno != 17:
+            raise
+    shutil.copy2(certout, fname)
+    hv = subprocess.check_output(
+        ['openssl', 'x509', '-in', certout, '-hash', '-noout'])
+    if not isinstance(hv, str):
+        hv = hv.decode('utf8')
+    hv = hv.strip()
+    hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
+    certname = '{0}.pem'.format(collective.get_myname())
+    for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
+        currname = os.path.join('/var/lib/confluent/public/site/tls/', currname)
+        if currname.endswith('.0'):
+            try:
+                realname = os.readlink(currname)
+                if realname == certname:
+                    os.unlink(currname)
+            except OSError:
+                pass
+    os.symlink(certname, hashname)
+
+if __name__ == '__main__':
+    outdir = os.getcwd()
+    keyout = os.path.join(outdir, 'key.pem')
+    certout = os.path.join(outdir, 'cert.pem')
+    create_certificate(keyout, certout)

confluent_server/confluent/config/attributes.py

@@ -97,6 +97,19 @@ node = {
         'description': ('Classification of node as server or switch'),
         'validvalues': ('switch', 'server'),
     },
+    'crypted.rootpassword': {
+        'description': 'The password of the local root password. '
+                       'This is stored as a non-recoverable hash.',
+    },
+    'crypted.selfapikey': {
+        'description': ('Crypt of api key for self api requests by node'),
+    },
+    'deployment.apiarmed': {
+        'description': ('Indicates whether the node authentication token interface '
+                        'is armed.  If set to once, it will grant only the next '
+                        'request. If set to continuous, will allow many requests.'
+                        'Should not be set unless an OS deployment is pending.'),
+    },
     #'id': {
     #    'description': ('Numeric identifier for node')
     #},
@@ -157,6 +170,30 @@ node = {
                         'indicates candidate managers, either for '
                         'high availability or load balancing purposes.')
     },
+    'deployment.pendingprofile': {
+        'description': ('An OS profile that is pending deployment.  This indicates to '
+                        'the network boot subsystem what should be offered when a potential '
+                        'network boot request comes in')
+    },
+    'deployment.profile': {
+        'description': ('The profile that has most recently reported '
+                        'completion of deployment. Note that an image may opt '
+                        'to leave itself both current and pending, for example '
+                        'a stateless profile would be both after first boot.')
+
+    },
+    'deployment.useinsecureprotocols': {
+        'description': ('What phase(s) of boot are permitted to use insecure protocols '
+                        '(TFTP and HTTP without TLS.  By default, HTTPS is allowed.  However '
+                        'this is not compatible with most firmware in most scenarios.  Using '
+                        '"firmware" as the setting will still use HTTPS after the initial download, '
+                        'though be aware that a successful compromise during the firmware phase '
+                        'will negate future TLS protections.  The value "always" will result in '
+                        'tftp/http being used for entire deployment.  Note that ONIE does not '
+                        'support secure protocols, and in that case this setting must be "always" '
+                        'or "firmware"'),
+        'validlist': ('always', 'firmware', 'never'),
+    },
     'discovery.passwordrules': {
         'description':  'Any specified rules shall be configured on the BMC '
                         'upon discovery.  "expiration=no,loginfailures=no,complexity=no,reuse=no" '
@@ -347,6 +384,19 @@ node = {
         'description': 'Whether or not the indicated network interface is to be used for booting.  This is used by '
                        'the discovery process to decide where to place the mac address of a detected PXE nic.',
     },
+    'net.ipv4_address': {
+        'description': 'When configuring static, use this address.  If '
+                       'unspecified, it will check if the node name resolves '
+                       'to an IP address.  Additionally, the subnet prefix '
+                       'may be specified with a suffix, e.g. "/16".  If not '
+                       'specified, it will attempt to autodetect based on '
+                       'current network configuration.'
+    },
+    'net.ipv4_method': {
+        'description': 'Whether to use static or dhcp when configuring this '
+                       'interface for IPv4.',
+        'validvalues': ('dhcp', 'static', 'none')
+    },
     'net.ipv4_gateway': {
         'description':  'The IPv4 gateway to use if applicable.  As is the '
                         'case for other net attributes, net.eth0.ipv4_gateway '
@@ -441,4 +491,10 @@ node = {
         'description': ('Fingerprint of the SSH key of the OS running on the '
                         'system.'),
     },
+    'dns.domain': {
+        'description': 'DNS Domain searched by default by the system'
+    },
+    'dns.servers': {
+        'description': 'DNS Server or servers to provide to node',
+    },
 }

confluent_server/confluent/config/configmanager.py

@@ -1,4 +1,4 @@
-7# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
 # Copyright 2015-2019 Lenovo
@@ -60,6 +60,7 @@ import confluent.util
 import confluent.netutil as netutil
 import confluent.exceptions as exc
 import copy
+import crypt
 try:
     import cPickle
 except ModuleNotFoundError:
@@ -112,6 +113,14 @@ _attraliases = {
 }
 _validroles = ('Administrator', 'Operator', 'Monitor')
 
+
+def attrib_supports_expression(attrib):
+    attrib = _attraliases.get(attrib, attrib)
+    if attrib.startswith('secret.') or attrib.startswith('crypted.'):
+        return False
+    return True
+
+
 def _mkpath(pathname):
     try:
         os.makedirs(pathname)
@@ -475,6 +484,13 @@ def _get_valid_attrname(attrname):
     return attrname
 
 
+def hashcrypt_value(value):
+    salt = confluent.util.stringify(base64.b64encode(os.urandom(12),
+                                    altchars=b'./'))
+    salt = '$6${0}'.format(salt)
+    return crypt.crypt(value, salt)
+
+
 def crypt_value(value,
                 key=None,
                 integritykey=None):
@@ -488,7 +504,8 @@ def crypt_value(value,
         key = _masterkey
     iv = os.urandom(12)
     crypter = AES.new(key, AES.MODE_GCM, nonce=iv)
-    value = confluent.util.stringify(value).encode('utf-8')
+    if not isinstance(value, bytes):
+        value = value.encode('utf-8')
     cryptval, hmac = crypter.encrypt_and_digest(value)
     return iv, cryptval, hmac, b'\x02'
 
@@ -1667,6 +1684,17 @@ class ConfigManager(object):
         self.set_group_attributes(attribmap, autocreate=True)
 
     def set_group_attributes(self, attribmap, autocreate=False):
+        for group in attribmap:
+            curr = attribmap[group]
+            for attrib in curr:
+                if attrib.startswith('crypted.'):
+                    if not isinstance(curr[attrib], dict):
+                        curr[attrib] = {'value': curr[attrib]}
+                    if 'hashvalue' not in curr[attrib]:
+                        curr[attrib]['hashvalue'] = hashcrypt_value(
+                            curr[attrib]['value'])
+                    if 'value' in curr[attrib]:
+                        del curr[attrib]['value']
         if cfgleader:  # currently config slave to another
             return exec_on_leader('_rpc_master_set_group_attributes',
                                   self.tenant, attribmap, autocreate)
@@ -1760,6 +1788,9 @@ class ConfigManager(object):
                 if 'value' in newdict and attr.startswith("secret."):
                     newdict['cryptvalue'] = crypt_value(newdict['value'])
                     del newdict['value']
+                if 'value' in newdict and attr.startswith("crypted."):
+                    newdict['hashvalue'] = hashcrypt_value(newdict['value'])
+                    del newdict['value']
                 cfgobj[attr] = newdict
                 if attr == 'nodes':
                     self._sync_nodes_to_group(group=group,
@@ -2072,6 +2103,17 @@ class ConfigManager(object):
 
 
     def set_node_attributes(self, attribmap, autocreate=False):
+        for node in attribmap:
+            curr = attribmap[node]
+            for attrib in curr:
+                if attrib.startswith('crypted.'):
+                    if not isinstance(curr[attrib], dict):
+                        curr[attrib] = {'value': curr[attrib]}
+                    if 'hashvalue' not in curr[attrib]:
+                        curr[attrib]['hashvalue'] = hashcrypt_value(
+                            curr[attrib]['value'])
+                    if 'value' in curr[attrib]:
+                        del curr[attrib]['value']
         if cfgleader:  # currently config slave to another
             return exec_on_leader('_rpc_master_set_node_attributes',
                                         self.tenant, attribmap, autocreate)
@@ -2162,6 +2204,9 @@ class ConfigManager(object):
                 if 'value' in newdict and attrname.startswith("secret."):
                     newdict['cryptvalue'] = crypt_value(newdict['value'])
                     del newdict['value']
+                if 'value' in newdict and attrname.startswith("crypted."):
+                    newdict['hashvalue'] = hashcrypt_value(newdict['value'])
+                    del newdict['value']
                 cfgobj[attrname] = newdict
                 if attrname == 'groups':
                     self._sync_groups_to_node(node=node,

confluent_server/confluent/consoleserver.py

@@ -108,10 +108,10 @@ def pytechars2line(chars, maxlen=None):
         char = chars[charidx]
         csi = bytearray([])
         if char.fg != lfg:
-            csi.append(30 + pytecolors2ansi[char.fg])
+            csi.append(30 + pytecolors2ansi.get(char.fg, 9))
             lfg = char.fg
         if char.bg != lbg:
-            csi.append(40 + pytecolors2ansi[char.bg])
+            csi.append(40 + pytecolors2ansi.get(char.bg, 9))
             lbg = char.bg
         if char.bold != lb:
             lb = char.bold
@@ -243,7 +243,7 @@ class ConsoleHandler(object):
     def check_collective(self, attrvalue):
         myc = attrvalue.get(self.node, {}).get('collective.manager', {}).get(
             'value', None)
-        if configmodule.list_collective() and not myc:
+        if list(configmodule.list_collective()) and not myc:
             self._is_local = False
             self._detach()
             self._disconnect()

confluent_server/confluent/core.py

@@ -46,6 +46,7 @@ import confluent.exceptions as exc
 import confluent.messages as msg
 import confluent.networking.macmap as macmap
 import confluent.noderange as noderange
+import confluent.osimage as osimage
 try:
     import confluent.shellmodule as shellmodule
 except ImportError:
@@ -69,6 +70,10 @@ import sys
 pluginmap = {}
 dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol')
 
+try:
+    unicode
+except NameError:
+    unicode = str
 
 def seek_element(currplace, currkey):
     try:
@@ -125,8 +130,9 @@ def load_plugins():
         sys.path.pop(1)
 
 
-rootcollections = ['discovery/', 'events/', 'networking/',
-                   'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' , 'users/', 'version']
+rootcollections = ['deployment/', 'discovery/', 'events/', 'networking/',
+                   'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' ,
+                   'users/', 'version']
 
 
 class PluginRoute(object):
@@ -138,6 +144,47 @@ class PluginCollection(object):
     def __init__(self, routedict):
         self.routeinfo = routedict
 
+
+def handle_deployment(configmanager, inputdata, pathcomponents,
+                      operation):
+    if len(pathcomponents) == 1:
+        yield msg.ChildCollection('distributions/')
+        yield msg.ChildCollection('profiles/')
+        yield msg.ChildCollection('importing/')
+        return
+    if pathcomponents[1] == 'distributions':
+        if len(pathcomponents) == 2:
+            for dist in osimage.list_distros():
+                yield msg.ChildCollection(dist + '/')
+            return
+    if pathcomponents[1] == 'profiles':
+        if len(pathcomponents) == 2:
+            for prof in osimage.list_profiles():
+                yield msg.ChildCollection(prof + '/')
+            return
+    if pathcomponents[1] == 'importing':
+        if len(pathcomponents) == 2 or not pathcomponents[-1]:
+            if operation == 'retrieve':
+                for imp in osimage.list_importing():
+                    yield imp
+                return
+            elif operation == 'create':
+                importer = osimage.MediaImporter(inputdata['filename'])
+                yield msg.KeyValueData({'target': importer.targpath,
+                                        'name': importer.importkey})
+                return
+        elif len(pathcomponents) == 3:
+            if operation == 'retrieve':
+                for res in osimage.get_importing_status(pathcomponents[-1]):
+                    yield res
+                return
+            elif operation == 'delete':
+                for res in osimage.remove_importing(pathcomponents[-1]):
+                    yield res
+                return
+    raise exc.NotFoundException('Unrecognized request')
+
+
 def _init_core():
     global noderesources
     global nodegroupresources
@@ -1056,10 +1103,6 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
         yield rsp
 
 
-def handle_discovery(pathcomponents, operation, configmanager, inputdata):
-    if pathcomponents[0] == 'detected':
-        pass
-
 def handle_discovery(pathcomponents, operation, configmanager, inputdata):
     if pathcomponents[0] == 'detected':
         pass
@@ -1080,6 +1123,9 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
     elif pathcomponents[0] == 'noderange':
         return handle_node_request(configmanager, inputdata, operation,
                                    pathcomponents, autostrip)
+    elif pathcomponents[0] == 'deployment':
+        return handle_deployment(configmanager, inputdata, pathcomponents,
+                                 operation)
     elif pathcomponents[0] == 'nodegroups':
         return handle_nodegroup_request(configmanager, inputdata,
                                         pathcomponents,

confluent_server/confluent/credserver.py

@@ -0,0 +1,84 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.config.configmanager as cfm
+import confluent.netutil as netutil
+import confluent.util as util
+import datetime
+import eventlet
+import eventlet.green.socket as socket
+import eventlet.greenpool
+import os
+
+class CredServer(object):
+    def __init__(self):
+        self.cfm = cfm.ConfigManager(None)
+
+    def handle_client(self, client, peer):
+        try:
+            if not netutil.address_is_local(peer[0]):
+                client.close()
+                return
+            client.send(b'\xc2\xd1-\xa8\x80\xd8j\xba')
+            tlv = bytearray(client.recv(2))
+            if tlv[0] != 1:
+                client.close()
+                return
+            nodename = util.stringify(client.recv(tlv[1]))
+            tlv = bytearray(client.recv(2))
+            apiarmed = self.cfm.get_node_attributes(nodename, 'deployment.apiarmed')
+            apiarmed = apiarmed.get(nodename, {}).get('deployment.apiarmed', {}).get(
+                'value', None)
+            if not apiarmed:
+                client.close()
+                return
+            if apiarmed not in ('once', 'continuous'):
+                now = datetime.datetime.utcnow()
+                expiry = datetime.datetime.strptime(apiarmed, "%Y-%m-%dT%H:%M:%SZ")
+                if now > expiry:
+                    self.cfm.set_node_attributes({nodename: {'deployment.apiarmed': ''}})
+                    client.close()
+                    return
+            client.send(b'\x02\x20')
+            rttoken = os.urandom(32)
+            client.send(rttoken)
+            client.send(b'\x00\x00')
+            tlv = bytearray(client.recv(2))
+            if tlv[0] != 3:
+                client.close()
+                return
+            echotoken = client.recv(tlv[1])
+            if echotoken != rttoken:
+                client.close()
+                return
+            tlv = bytearray(client.recv(2))
+            if tlv[0] != 4:
+                client.close()
+                return
+            echotoken = util.stringify(client.recv(tlv[1]))
+            cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': echotoken}, 'deployment.apiarmed': ''}}
+            if apiarmed == 'continuous':
+                del cfgupdate[nodename]['deployment.apiarmed']
+            self.cfm.set_node_attributes(cfgupdate)
+            client.recv(2)  # drain end of message
+            client.send(b'\x05\x00') # report success
+        finally:
+            client.close()
+
+if __name__ == '__main__':
+    a = CredServer()
+    while True:
+        eventlet.sleep(86400)

confluent_server/confluent/discovery/core.py

@@ -704,11 +704,11 @@ def detected(info):
     if nodename and handler:
         eval_node(cfg, handler, info, nodename)
     elif handler:
-        log.log(
-            {'info': 'Detected unknown {0} with hwaddr {1} at '
-                     'address {2}'.format(
-                        handler.devname, info['hwaddr'], handler.ipaddr
-                      )})
+        #log.log(
+        #    {'info': 'Detected unknown {0} with hwaddr {1} at '
+        #             'address {2}'.format(
+        #                handler.devname, info['hwaddr'], handler.ipaddr
+        #              )})
         info['discostatus'] = 'unidentified'
         unknown_info[info['hwaddr']] = info
 
@@ -877,6 +877,8 @@ def get_nodename_from_chained_smms(cfg, handler, info):
                 nodename = newnodename
     return nodename
 
+def get_node_by_uuid(uuid):
+    return nodes_by_uuid.get(uuid, None)
 
 def get_nodename_from_enclosures(cfg, info):
     nodename = None
@@ -1076,7 +1078,7 @@ def discover_node(cfg, handler, info, nodename, manual):
                 traceback.print_exc()
                 return False
             newnodeattribs = {}
-            if cfm.list_collective():
+            if list(cfm.list_collective()):
                 # We are in a collective, check collective.manager
                 cmc = cfg.get_node_attributes(nodename, 'collective.manager')
                 cm = cmc.get(nodename, {}).get('collective.manager', {}).get('value', None)
@@ -1140,8 +1142,8 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
         return True
     if uuid_is_valid(info['uuid']):
         known_pxe_uuids[info['uuid']] = nodename
-    log.log({'info': 'Detected {0} ({1} with mac {2})'.format(
-        nodename, handler.devname, info['hwaddr'])})
+    #log.log({'info': 'Detected {0} ({1} with mac {2})'.format(
+    #    nodename, handler.devname, info['hwaddr'])})
     return True
 
 
@@ -1246,8 +1248,7 @@ def start_detection():
     if rechecker is None:
         rechecktime = util.monotonic_time() + 900
         rechecker = eventlet.spawn_after(900, _periodic_recheck, cfg)
-
-    # eventlet.spawn_n(ssdp.snoop, safe_detected)
+    eventlet.spawn_n(ssdp.snoop, None, None, ssdp, get_node_by_uuid)
 
 def stop_autosense():
     for watcher in list(autosensors):

confluent_server/confluent/discovery/protocols/pxe.py

@@ -22,18 +22,141 @@
 
 # option 97 = UUID (wireformat)
 
+import confluent.config.configmanager as cfm
+import confluent.log as log
+import confluent.netutil as netutil
+import ctypes
+import ctypes.util
+import eventlet
 import eventlet.green.socket as socket
+import eventlet.green.select as select
+import netifaces
 import struct
 
+libc = ctypes.CDLL(ctypes.util.find_library('c'))
+
+iphdr = b'\x45\x00\x00\x00\x00\x00\x00\x00\x40\x11\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff'
+constiphdrsum = b'\x85\x11'
+udphdr = b'\x00\x43\x00\x44\x00\x00\x00\x00'
+
+def _ipsum(data):
+    currsum = 0
+    if len(data) % 2:
+        currsum = struct.unpack('!B', data[-1:])[0] << 8
+        data = memoryview(data)
+        data = data[:-1]
+    for datum in struct.unpack('!' + 'H' * (len(data) // 2), data):
+        currsum += datum
+        if currsum >> 16:
+            currsum &= 0xffff
+            currsum += 1
+    if currsum == 0:
+        currsum = 0xffff
+    return currsum
+
+class sockaddr_ll(ctypes.Structure):
+    _fields_ = [('sll_family', ctypes.c_ushort),
+                ('sll_protocol', ctypes.c_ushort),
+                ('sll_ifindex', ctypes.c_int),
+                ('sll_hatype', ctypes.c_ushort),
+                ('sll_pkttype', ctypes.c_ubyte),
+                ('sll_halen', ctypes.c_ubyte),
+                ('sll_addr', ctypes.c_ubyte * 20)]
+
+class iovec(ctypes.Structure):   # from uio.h
+    _fields_ = [('iov_base', ctypes.c_void_p),
+                ('iov_len', ctypes.c_size_t)]
+
+class msghdr(ctypes.Structure):  # from bits/socket.h
+    _fields_ = [('msg_name', ctypes.c_void_p),
+                ('msg_namelen', ctypes.c_uint),
+                ('msg_iov', ctypes.POINTER(iovec)),
+                ('msg_iovlen', ctypes.c_size_t),
+                ('msg_control', ctypes.c_void_p),
+                ('msg_controllen', ctypes.c_size_t),
+                ('msg_flags', ctypes.c_int)]
+
+class cmsghdr(ctypes.Structure):  # also from bits/socket.h
+    _fields_ = [('cmsg_len', ctypes.c_size_t),
+                ('cmsg_level', ctypes.c_int),
+                ('cmsg_type', ctypes.c_int)]
+                # ignore the __extension__
+
+class in_addr(ctypes.Structure):
+    _fields_ = [('s_addr', ctypes.c_uint32)]
+
+class in_pktinfo(ctypes.Structure):  # from bits/in.h
+    _fields_ = [('ipi_ifindex', ctypes.c_int),
+                ('ipi_spec_dst', in_addr),
+                ('ipi_addr', in_addr)]
+
+class sockaddr_in(ctypes.Structure):
+    _fields_ = [('sin_family', ctypes.c_ushort),  # per bits/sockaddr.h
+                ('sin_port', ctypes.c_uint16),  # per netinet/in.h
+                ('sin_addr', in_addr)]
+
+
+sendto = libc.sendto
+sendto.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t,
+                   ctypes.c_int, ctypes.POINTER(sockaddr_ll),
+                   ctypes.c_size_t]
+sendto.restype = ctypes.c_size_t
+recvmsg = libc.recvmsg
+recvmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+recvmsg.restype = ctypes.c_size_t
+
+pkttype = ctypes.c_char * 2048
+
+_idxtoname = libc.if_indextoname
+_idxtoname.argtypes = [ctypes.c_uint, ctypes.c_char_p]
+
+def idxtoname(idx):
+    name = (ctypes.c_char * 16)()
+    _idxtoname(idx, name)
+    ret = name.value.strip()
+    if not isinstance(ret, str):
+        ret = ret.decode('utf8')
+    return ret
+
+_idxtobcast = {}
+def get_bcastaddr(idx):
+    if idx not in _idxtobcast:
+        bc = netifaces.ifaddresses(idxtoname(idx))[17][0]['broadcast']
+        bc = bytearray([int(x, 16) for x in bc.split(':')])
+        _idxtobcast[idx] = bc
+    return _idxtobcast[idx]
+
+
+IP_PKTINFO = 8
+
+
+def CMSG_ALIGN(length):  # bits/socket.h
+    ret = (length + ctypes.sizeof(ctypes.c_size_t) - 1
+           & ~(ctypes.sizeof(ctypes.c_size_t) - 1))
+    return ctypes.c_size_t(ret)
+
+
+def CMSG_SPACE(length):  # bits/socket.h
+    ret = CMSG_ALIGN(length).value + CMSG_ALIGN(ctypes.sizeof(cmsghdr)).value
+    return ctypes.c_size_t(ret)
+
+
+cmsgtype = ctypes.c_char * CMSG_SPACE(ctypes.sizeof(in_pktinfo)).value
+cmsgsize = CMSG_SPACE(ctypes.sizeof(in_pktinfo)).value
+
 pxearchs = {
-    '\x00\x00': 'bios-x86',
-    '\x00\x07': 'uefi-x64',
-    '\x00\x09': 'uefi-x64',
-    '\x00\x0b': 'uefi-aarch64',
-    '\x00\x10': 'uefi-httpboot',
+    b'\x00\x00': 'bios-x86',
+    b'\x00\x07': 'uefi-x64',
+    b'\x00\x09': 'uefi-x64',
+    b'\x00\x0b': 'uefi-aarch64',
+    b'\x00\x10': 'uefi-httpboot',
 }
 
 
+uuidmap = {}
+macmap = {}
+attribwatcher = None
+
 def stringify(value):
     string = bytes(value)
     if not isinstance(string, str):
@@ -60,109 +183,415 @@ def _decode_ocp_vivso(rq, idx, size):
         idx += rq[idx + 1] + 2
     return '', None, vivso
 
-
-def find_info_in_options(rq, optidx):
-    uuid = None
-    arch = None
-    vivso = None
-    ztpurlrequested = False
-    iscumulus = False
+def opts_to_dict(rq, optidx, expectype=1):
+    reqdict = {}
+    disco = {'uuid':None, 'arch': None, 'vivso': None}
     try:
-        while uuid is None or arch is None:
-            if rq[optidx] == 53:  # DHCP message type
-                # we want only length 1 and only discover (type 1)
-                if rq[optidx + 1] != 1 or rq[optidx + 2] != 1:
-                    return uuid, arch, vivso
-                optidx += 3
-            elif rq[optidx] == 55:
-                if 239 in rq[optidx + 2:optidx + 2 + rq[optidx + 1]]:
-                    ztpurlrequested = True
-                optidx += rq[optidx + 1] + 2
-            elif rq[optidx] == 60:
-                vci = stringify(rq[optidx + 2:optidx + 2 + rq[optidx + 1]])
-                if vci.startswith('cumulus-linux'):
-                    iscumulus = True
-                    arch = vci.replace('cumulus-linux', '').strip()
-                optidx += rq[optidx + 1] + 2
-            elif rq[optidx] == 97:
-                if rq[optidx + 1] != 17:
-                    # 16 bytes of uuid and one reserved byte
-                    return uuid, arch, vivso
-                if rq[optidx + 2] != 0:  # the reserved byte should be zero,
-                    # anything else would be a new spec that we don't know yet
-                    return uuid, arch, vivso
-                uuid = decode_uuid(rq[optidx + 3:optidx + 19])
-                optidx += 19
-            elif rq[optidx] == 93:
-                if rq[optidx + 1] != 2:
-                    return uuid, arch
-                archraw = bytes(rq[optidx + 2:optidx + 4])
-                if archraw in pxearchs:
-                    arch = pxearchs[archraw]
-                optidx += 4
-            elif rq[optidx] == 125:
-                #vivso = rq[optidx + 2:optidx + 2 + rq[optidx + 1]]
-                if rq[optidx + 2:optidx + 6] == b'\x00\x00\xa6\x7f':  # OCP
-                    return _decode_ocp_vivso(rq, optidx + 7, rq[optidx + 6])
-                optidx += rq[optidx + 1] + 2
-            else:
-                optidx += rq[optidx + 1] + 2
+        while optidx < len(rq):
+            optnum = rq[optidx]
+            optlen = rq[optidx + 1]
+            reqdict[optnum] = rq[optidx + 2:optidx + 2 + optlen]
+            optidx += optlen + 2
     except IndexError:
         pass
-    if not vivso and iscumulus and ztpurlrequested:
-        if not uuid:
-            uuid = ''
-        vivso = {'service-type': 'cumulus-switch', 'arch': arch}
-    return uuid, arch, vivso
+    if reqdict.get(53, [0])[0] != expectype:
+        return reqdict, disco
+    # It is a discover packet..
+    iscumulus = False
+    maybeztp = False
+    if 239 in reqdict.get(55, []):
+        maybeztp = True
+    vci = stringify(reqdict.get(60, b''))
+    if vci.startswith('cumulus-linux'):
+        disco['arch'] = vci.replace('cumulus-linux', '').strip()
+        iscumulus = True
+    if reqdict.get(93, None):
+        disco['arch'] = pxearchs.get(bytes(reqdict[93]), None)
+    if reqdict.get(97, None):
+        uuidcandidate = reqdict[97]
+        if uuidcandidate[0] != 0:
+            return reqdict, disco
+        disco['uuid'] = decode_uuid(uuidcandidate[1:])
+    if reqdict.get(125, None):
+        if reqdict[125][:4] == b'\x00\x00\xa6\x7f':  # OCP
+            disco['vivso'] = _decode_ocp_vivso(
+                reqdict[125], 5, reqdict[125][4])[-1]
+            return reqdict, disco
+    if not disco['vivso'] and iscumulus and maybeztp:
+        if not disco['uuid']:
+            disco['uuid'] = ''
+        disco['vivso'] = {'service-type': 'cumulus-switch',
+                          'arch': disco['arch']}
+    return reqdict, disco
+
+
+def ipfromint(numb):
+    return socket.inet_ntoa(struct.pack('I', numb))
+
+def proxydhcp():
+    net4011 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net4011.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4011.setsockopt(socket.IPPROTO_IP, IP_PKTINFO, 1)
+    net4011.bind(('', 4011))
+    cfg = cfm.ConfigManager(None)
+    while True:
+        ready = select.select([net4011], [], [], None)
+        if not ready or not ready[0]:
+            continue
+        rq = bytearray(1024)
+        rqv = memoryview(rq)
+        nb, client = net4011.recvfrom_into(rq)
+        if nb < 240:
+            continue
+        rp = bytearray(1024)
+        rpv = memoryview(rp)
+        try:
+            optidx = rq.index(b'\x63\x82\x53\x63') + 4
+        except ValueError:
+            continue
+        opts, disco = opts_to_dict(rq, optidx, 3)
+        disco['uuid']
+        node = None
+        if disco.get('hwaddr', None) in macmap:
+            node = macmap[disco['hwaddr']]
+        elif disco.get('uuid', None) in uuidmap:
+            node = uuidmap[disco['uuid']]
+        if not node:
+            continue
+        hwlen = rq[2]
+        myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
+        if not myipn:
+            continue
+        if opts.get(77, None) == b'iPXE':
+            cfd = cfg.get_node_attributes(node, ('deployment.*'))
+            profile = cfd.get(node, {}).get(
+                'deployment.pendingprofile', {}).get('value', None)
+            if not profile:
+                continue
+            myip = socket.inet_ntoa(myipn)
+            bootfile = 'http://{0}/confluent-public/os/{1}/boot/boot.ipxe'.format(myip, profile).encode('utf8')
+        elif disco['arch'] == 'uefi-x64':
+            bootfile = b'confluent/x86_64/ipxe.efi'
+        elif disco['arch'] == 'bios-x86':
+            bootfile = b'confluent/x86_64/ipxe.kkpxe'
+        rpv[:240] = rqv[:240].tobytes()
+        rpv[0:1] = b'\x02'
+        rpv[108:108 + len(bootfile)] = bootfile
+        rpv[240:243] = b'\x35\x01\x05'
+        rpv[243:249] = b'\x36\x04' + myipn
+        rpv[20:24] = myipn
+        rpv[249:268] = b'\x61\x11' + opts[97]
+        rpv[268:280] = b'\x3c\x09PXEClient\xff'
+        net4011.sendto(rpv[:281], client)
+
+
+def start_proxydhcp():
+    eventlet.spawn_n(proxydhcp)
+
 
 def snoop(handler, protocol=None):
     #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
     #prominent
-    #TODO(jjohnson2): IP_PKTINFO, recvmsg to get the destination ip, per
-    #proxydhcp.c from xCAT
+    #TODO(jjohnson2): enable unicast replies. This would suggest either
+    # injection into the neigh table before OFFER or using SOCK_RAW.
+    global attribwatcher
+    cfg = cfm.ConfigManager(None)
+    remap_nodes(cfg.list_nodes(), cfg)
+    attribwatcher = cfg.watch_attributes(cfg.list_nodes(), ('id.uuid', 'net.*hwaddr'), remap_nodes)
+    cfg.watch_nodecollection(new_nodes)
     net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+    net4.setsockopt(socket.IPPROTO_IP, IP_PKTINFO, 1)
     net4.bind(('', 67))
     while True:
         # Just need some delay, picked a prime number so that overlap with other
         # timers might be reduced, though it really is probably nothing
-        (rq, peer) = net4.recvfrom(9000)
+        ready = select.select([net4], [], [], None)
+        if not ready or not ready[0]:
+            continue
+        clientaddr = sockaddr_in()
+        rawbuffer = bytearray(2048)
+        data = pkttype.from_buffer(rawbuffer)
+        msg = msghdr()
+        cmsgarr = bytearray(cmsgsize)
+        cmsg = cmsgtype.from_buffer(cmsgarr)
+        iov = iovec()
+        iov.iov_base = ctypes.addressof(data)
+        iov.iov_len = 2048
+        msg.msg_iov = ctypes.pointer(iov)
+        msg.msg_iovlen = 1
+        msg.msg_control = ctypes.addressof(cmsg)
+        msg.msg_controllen = ctypes.sizeof(cmsg)
+        msg.msg_name = ctypes.addressof(clientaddr)
+        msg.msg_namelen = ctypes.sizeof(clientaddr)
+        # We'll leave name and namelen blank for now
+        i = recvmsg(net4.fileno(), ctypes.pointer(msg), 0)
         # if we have a small packet, just skip, it can't possible hold enough
         # data and avoids some downstream IndexErrors that would be messy
         # with try/except
-        if len(rq) < 64:
+        if i < 64:
             continue
-        rq = bytearray(rq)
+        #peer = ipfromint(clientaddr.sin_addr.s_addr)
+        # We don't need peer yet, generally it's 0.0.0.0
+        _, level, typ = struct.unpack('QII', cmsgarr[:16])
+        if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
+            idx, recv, targ = struct.unpack('III', cmsgarr[16:28])
+            recv = ipfromint(recv)
+            targ = ipfromint(targ)
+        # peer is the source ip (in dhcpdiscover, 0.0.0.0)
+        # recv is the 'ip' that recevied the packet, regardless of target
+        # targ is the ip in the destination ip of the header.
+        # idx is the ip link number of the receiving nic
+        # For example, a DHCPDISCOVER will probably have:
+        # peer of 0.0.0.0
+        # targ of 255.255.255.255
+        # recv of <actual ip address that could reply>
+        # idx correlated to the nic
+        rqv = memoryview(rawbuffer)
+        rq = bytearray(rqv[:i])
         if rq[0] == 1:  # Boot request
             addrlen = rq[2]
             if addrlen > 16 or addrlen == 0:
                 continue
-            netaddr = rq[28:28+addrlen]
-            netaddr = ':'.join(['{0:02x}'.format(x) for x in netaddr])
+            rawnetaddr = rq[28:28+addrlen]
+            netaddr = ':'.join(['{0:02x}'.format(x) for x in rawnetaddr])
             optidx = 0
             try:
                 optidx = rq.index(b'\x63\x82\x53\x63') + 4
             except ValueError:
                 continue
-            uuid, arch, vivso = find_info_in_options(rq, optidx)
+            txid = rq[4:8] # struct.unpack('!I', rq[4:8])[0]
+            rqinfo, disco = opts_to_dict(rq, optidx)
+            vivso = disco.get('vivso', None)
             if vivso:
                 # info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
-                handler({'hwaddr': netaddr, 'uuid': uuid,
-                         'architecture': vivso.get('arch', ''),
-                         'services': (vivso['service-type'],),
-                         'attributes': {'enclosure-machinetype-model': [vivso.get('machine', '')]}})
-                continue
-            if uuid is None:
+                info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                        'architecture': vivso.get('arch', ''),
+                        'services': (vivso['service-type'],),
+                        'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid},
+                        'attributes': {'enclosure-machinetype-model': [vivso.get('machine', '')]}}
+                handler(info)
+                #consider_discover(info, rqinfo, net4, cfg, rqv)
                 continue
             # We will fill out service to have something to byte into,
             # but the nature of the beast is that we do not have peers,
             # so that will not be present for a pxe snoop
-            handler({'hwaddr': netaddr, 'uuid': uuid, 'architecture': arch,
-                     'services': ('pxe-client',)})
+            info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                    'architecture': disco['arch'],
+                    'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid},
+                    'services': ('pxe-client',)}
+            if disco['uuid']:  #TODO(jjohnson2): need to explictly check for
+                               # discover, so that the parser can go ahead and
+                               # parse the options including uuid to enable
+                               # ACK
+                handler(info)
+            consider_discover(info, rqinfo, net4, cfg, rqv)
+
+
+
+def clear_nodes(nodes):
+    for nodename in nodes:
+        for ent in list(macmap):
+            if macmap[ent] == nodename:
+                del macmap[ent]
+        for ent in list(uuidmap):
+            if uuidmap[ent] == nodename:
+                del uuidmap[ent]
+
+
+def new_nodes(added, deleting, renamed, configmanager):
+    global attribwatcher
+    configmanager.remove_watcher(attribwatcher)
+    alldeleting = set(deleting) | set(renamed)
+    clear_nodes(alldeleting)
+    attribwatcher = configmanager.watch_attributes(configmanager.list_nodes(),
+                                                   ('id.uuid', 'net.*hwaddr'), remap_nodes)
+
+
+def remap_nodes(nodeattribs, configmanager):
+    global macmap
+    global uuidmap
+    updates = configmanager.get_node_attributes(nodeattribs, ('id.uuid', 'net.*hwaddr'))
+    clear_nodes(nodeattribs)
+    for node in updates:
+        for attrib in updates[node]:
+            if attrib == 'id.uuid':
+                uuidmap[updates[node][attrib]['value']] = node
+            elif 'hwaddr' in attrib:
+                macmap[updates[node][attrib]['value']] = node
+
+
+staticassigns = {}
+myipbypeer = {}
+def check_reply(node, info, packet, sock, cfg, reqview):
+    httpboot = info['architecture'] == 'uefi-httpboot'
+    replen = 275  # default is going to be 286
+    cfd = cfg.get_node_attributes(node, ('deployment.*'))
+    profile = cfd.get(node, {}).get('deployment.pendingprofile', {}).get('value', None)
+    myipn = info['netinfo']['recvip']
+    myipn = socket.inet_aton(myipn)
+    if not profile:
+        return
+    rqtype = packet[53][0]
+    insecuremode = cfd.get(node, {}).get('deployment.useinsecureprotocols',
+        'never')
+    if not insecuremode:
+        insecuremode = 'never'
+    if insecuremode == 'never' and not httpboot:
+        if rqtype == 1 and info['architecture']:
+            log.log(
+                {'info': 'Boot attempt by {0} detected in insecure mode, but '
+                        'insecure mode is disabled.  Set the attribute '
+                        '`deployment.useinsecureprotocols` to `firmware` or '
+                        '`always` to enable support, or use UEFI HTTP boot '
+                        'with HTTPS.'.format(node)})
+        return
+    reply = bytearray(512)
+    repview = memoryview(reply)
+    repview[:20] = iphdr
+    repview[12:16] = myipn
+    repview[20:28] = udphdr
+    repview = repview[28:]
+    repview[0:1] = b'\x02'
+    repview[1:10] = reqview[1:10] # duplicate txid, hwlen, and others
+    repview[10:11] = b'\x80'  # always set broadcast
+    repview[28:44] = reqview[28:44]  # copy chaddr field
+    if httpboot:
+        proto = 'https' if insecuremode == 'never' else 'http'
+        bootfile = '{0}://{1}/confluent-public/os/{2}/boot.img'.format(
+            proto, info['netinfo']['recvip'], profile
+        )
+        if not isinstance(bootfile, bytes):
+            bootfile = bootfile.encode('utf8')
+        repview[108:108 + len(bootfile)] = bootfile
+    repview[20:24] = myipn
+    gateway = None
+    netmask = None
+    niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx'])
+    if niccfg.get('ipv4_broken', False):
+        # Received a request over a nic with no ipv4 configured, ignore it
+        return
+    clipn = None
+    if niccfg['ipv4_address']:
+        clipn = socket.inet_aton(niccfg['ipv4_address'])
+        repview[16:20] = clipn
+        gateway = niccfg['ipv4_gateway']
+        if gateway:
+            gateway = socket.inet_aton(gateway)
+        netmask = niccfg['prefix']
+        netmask = (2**32 - 1) ^ (2**(32 - netmask) - 1)
+        netmask = struct.pack('!I', netmask)
+    repview[236:240] = b'\x63\x82\x53\x63'
+    repview[240:242] = b'\x35\x01'
+    if rqtype == 1:  # if discover, then offer
+        repview[242:243] = b'\x02'
+    elif rqtype == 3: # if request, then ack
+        repview[242:243] = b'\x05'
+    repview[243:245] = b'\x36\x04' # DHCP server identifier
+    repview[245:249] = myipn
+    repview[249:255] = b'\x33\x04\x00\x00\x00\xf0'  # fixed short lease time
+    repview[255:257] = b'\x61\x11'
+    repview[257:274] = packet[97]
+    # Note that sending PXEClient kicks off the proxyDHCP procedure, ignoring
+    # boot filename and such in the DHCP packet
+    # we will simply always do it to provide the boot payload in a consistent
+    # matter to both dhcp-elsewhere and fixed ip clients
+    if info['architecture'] == 'uefi-httpboot':
+        repview[replen - 1:replen + 11] = b'\x3c\x0aHTTPClient'
+        replen += 12
+    else:
+        repview[replen - 1:replen + 10] = b'\x3c\x09PXEClient'
+        replen += 11
+    hwlen = bytearray(reqview[2:3].tobytes())[0]
+    fulladdr = repview[28:28+hwlen].tobytes()
+    myipbypeer[fulladdr] = myipn
+    if hwlen == 8: # omnipath may present a mangled proxydhcp request later
+        shortaddr = bytearray(6)
+        shortaddr[0] = 2
+        shortaddr[1:] = fulladdr[3:]
+        myipbypeer[bytes(shortaddr)] = myipn
+    if netmask:
+        repview[replen - 1:replen + 1] = b'\x01\x04'
+        repview[replen + 1:replen + 5] = netmask
+        replen += 6
+    if gateway:
+        repview[replen - 1:replen + 1] = b'\x03\x04'
+        repview[replen + 1:replen + 5] = gateway
+        replen += 6
+    repview[replen - 1:replen] = b'\xff'  # end of options, should always be last byte
+    repview = memoryview(reply)
+    pktlen = struct.pack('!H', replen + 28)  # ip+udp = 28
+    repview[2:4] = pktlen
+    curripsum = ~(_ipsum(constiphdrsum + pktlen + myipn)) & 0xffff
+    repview[10:12] = struct.pack('!H', curripsum)
+    repview[24:26] = struct.pack('!H', replen + 8)
+    datasum = _ipsum(b'\x00\x11' + repview[24:26].tobytes() +
+                     repview[12:replen + 28].tobytes())
+    datasum = ~datasum & 0xffff
+    repview[26:28] = struct.pack('!H', datasum)
+    if clipn:
+        staticassigns[fulladdr] = (clipn, repview[:replen + 28].tobytes())
+    elif fulladdr in staticassigns:
+        del staticassigns[fulladdr]
+    send_raw_packet(repview, replen + 28, reqview, info)
+
+def send_raw_packet(repview, replen, reqview, info):
+    ifidx = info['netinfo']['ifidx']
+    tsock = socket.socket(socket.AF_PACKET, socket.SOCK_DGRAM,
+                          socket.htons(0x800))
+    targ = sockaddr_ll()
+    bcastaddr = get_bcastaddr(ifidx)
+    hwlen = len(bcastaddr)
+    bcastaddr20 = bytearray(20)
+    bcastaddr20[:hwlen] = bcastaddr
+    targ.sll_addr = (ctypes.c_ubyte * 20).from_buffer(bcastaddr20)
+    targ.sll_family = socket.AF_PACKET
+    targ.sll_halen = hwlen
+    targ.sll_protocol = socket.htons(0x800)
+    targ.sll_ifindex = ifidx
+    try:
+        pkt = ctypes.byref((ctypes.c_char * (replen)).from_buffer(repview))
+    except TypeError:
+        # Python 2....
+        pkt = ctypes.byref((ctypes.c_char * (replen)).from_buffer_copy(
+            repview[:replen].tobytes()))
+    sendto(tsock.fileno(), pkt, replen, 0, ctypes.byref(targ),
+           ctypes.sizeof(targ))
+
+def ack_request(pkt, rq, info):
+    hwlen = bytearray(rq[2:3].tobytes())[0]
+    hwaddr = rq[28:28+hwlen].tobytes()
+    myipn = myipbypeer.get(hwaddr, None)
+    if not myipn or pkt.get(54, None) != myipn:
+        return
+    assigninfo = staticassigns.get(hwaddr, None)
+    if assigninfo == None:
+        return
+    if pkt.get(50, None) != assigninfo[0]:
+        return
+    rply = assigninfo[1]
+    reply = bytearray(512)
+    repview = memoryview(reply)
+    repview[:len(rply)] = rply
+    repview[270:271] = b'\x05'
+    repview[26:28] = struct.pack('!H', 0) # TODO: use datasum, it was incorrect)
+    datasum = _ipsum(b'\x00\x11' + repview[24:26].tobytes() +
+                     repview[12:len(rply)].tobytes())
+    datasum = ~datasum & 0xffff
+    repview[26:28] = struct.pack('!H', datasum)
+    send_raw_packet(repview, len(rply), rq, info)
+
+def consider_discover(info, packet, sock, cfg, reqview):
+    if info.get('hwaddr', None) in macmap and info.get('uuid', None):
+        check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview)
+    elif info.get('uuid', None) in uuidmap:
+        check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview)
+    elif packet.get(53, None) == b'\x03':
+        ack_request(packet, reqview, info)
+
 
 if __name__ == '__main__':
     def testsnoop(info):
         print(repr(info))
-    snoop(testsnoop)
-
-
+    snoop(testsnoop)

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -30,13 +30,16 @@
 
 import confluent.neighutil as neighutil
 import confluent.util as util
+import confluent.log as log
 import eventlet.green.select as select
 import eventlet.green.socket as socket
+import time
 try:
     from eventlet.green.urllib.request import urlopen
 except (ImportError, AssertionError):
     from eventlet.green.urllib2 import urlopen
 import struct
+import traceback
 
 mcastv4addr = '239.255.255.250'
 mcastv6addr = 'ff02::c'
@@ -69,7 +72,7 @@ def scan(services, target=None):
             yield rply
 
 
-def snoop(handler, byehandler=None):
+def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
     """Watch for SSDP notify messages
 
     The handler shall be called on any service coming online.
@@ -85,6 +88,7 @@ def snoop(handler, byehandler=None):
     # Normally, I like using v6/v4 agnostic socket. However, since we are
     # dabbling in multicast wizardry here, such sockets can cause big problems,
     # so we will have two distinct sockets
+    tracelog = log.Logger('trace')
     known_peers = set([])
     net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
     net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
@@ -108,50 +112,82 @@ def snoop(handler, byehandler=None):
     net6.bind(('', 1900))
     peerbymacaddress = {}
     while True:
-        newmacs = set([])
-        machandlers = {}
-        r, _, _ = select.select((net4, net6), (), (), 60)
-        neighutil.update_neigh()
-        while r:
-            for s in r:
-                (rsp, peer) = s.recvfrom(9000)
-                rsp = rsp.split('\r\n')
-                method, _, _ = rsp[0].split(' ', 2)
-                if method == 'NOTIFY':
-                    ip = peer[0].partition('%')[0]
-                    if ip not in neighutil.neightable:
+        try:
+            newmacs = set([])
+            machandlers = {}
+            r, _, _ = select.select((net4, net6), (), (), 60)
+            neighutil.update_neigh()
+            while r:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    if rsp[:4] == b'PING':
                         continue
-                    if peer in known_peers:
-                        continue
-                    mac = neighutil.neightable[ip]
-                    known_peers.add(peer)
-                    newmacs.add(mac)
-                    if mac in peerbymacaddress:
-                        peerbymacaddress[mac]['addresses'].append(peer)
-                    else:
-                        peerbymacaddress[mac] = {
-                            'hwaddr': mac,
-                            'addresses': [peer],
-                        }
-                        peerdata = peerbymacaddress[mac]
+                    rsp = rsp.split(b'\r\n')
+                    method, _, _ = rsp[0].split(b' ', 2)
+                    if method == b'NOTIFY':
+                        ip = peer[0].partition('%')[0]
+                        if ip not in neighutil.neightable:
+                            continue
+                        if peer in known_peers:
+                            continue
+                        mac = neighutil.neightable[ip]
+                        known_peers.add(peer)
+                        newmacs.add(mac)
+                        if mac in peerbymacaddress:
+                            peerbymacaddress[mac]['addresses'].append(peer)
+                        else:
+                            peerbymacaddress[mac] = {
+                                'hwaddr': mac,
+                                'addresses': [peer],
+                            }
+                            peerdata = peerbymacaddress[mac]
+                            for headline in rsp[1:]:
+                                if not headline:
+                                    continue
+                                headline = util.stringify(headline)
+                                header, _, value = headline.partition(':')
+                                header = header.strip()
+                                value = value.strip()
+                                if header == 'NT':
+                                    peerdata['service'] = value
+                                elif header == 'NTS':
+                                    if value == 'ssdp:byebye':
+                                        machandlers[mac] = byehandler
+                                    elif value == 'ssdp:alive':
+                                        machandlers[mac] = None # handler
+                    elif method == b'M-SEARCH':
+                        if not uuidlookup:
+                            continue
+                        #ip = peer[0].partition('%')[0]
                         for headline in rsp[1:]:
                             if not headline:
                                 continue
-                            header, _, value = headline.partition(':')
-                            header = header.strip()
-                            value = value.strip()
-                            if header == 'NT':
-                                peerdata['service'] = value
-                            elif header == 'NTS':
-                                if value == 'ssdp:byebye':
-                                    machandlers[mac] = byehandler
-                                elif value == 'ssdp:alive':
-                                    machandlers[mac] = handler
-            r, _, _ = select.select((net4, net6), (), (), 0.1)
-        for mac in newmacs:
-            thehandler = machandlers.get(mac, None)
-            if thehandler:
-                thehandler(peerbymacaddress[mac])
+                            headline = util.stringify(headline)
+                            headline = headline.partition(':')
+                            if len(headline) < 3:
+                                continue
+                            if  headline[0] == 'ST' and headline[-1].startswith(' urn:xcat.org:service:confluent:'):
+                                for query in headline[-1].split('/'):
+                                    if query.startswith('uuid='):
+                                        curruuid = query.split('=', 1)[1].lower()
+                                        node = uuidlookup(curruuid)
+                                        if not node:
+                                            break
+                                        currtime = time.time()
+                                        seconds = int(currtime)
+                                        msecs = int(currtime * 1000 % 1000)
+                                        reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}\r\nCURRTIME: {1}\r\nCURRMSECS: {2}\r\n'.format(node, seconds, msecs)
+                                        if not isinstance(reply, bytes):
+                                            reply = reply.encode('utf8')
+                                        s.sendto(reply, peer)
+                r, _, _ = select.select((net4, net6), (), (), 0.2)
+            for mac in newmacs:
+                thehandler = machandlers.get(mac, None)
+                if thehandler:
+                    thehandler(peerbymacaddress[mac])
+        except Exception:
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)
 
 
 def _find_service(service, target):
@@ -163,18 +199,26 @@ def _find_service(service, target):
         for addr in addrs:
             host = addr[4][0]
             if addr[0] == socket.AF_INET:
-                net4.sendto(smsg.format(host, service), addr[4])
+                msg = smsg.format(host, service)
+                if not isinstance(msg, bytes):
+                    msg = msg.encode('utf8')
+                net4.sendto(msg, addr[4])
             elif addr[0] == socket.AF_INET6:
                 host = '[{0}]'.format(host)
-                net6.sendto(smsg.format(host, service), addr[4])
+                msg = smsg.format(host, service)
+                if not isinstance(msg, bytes):
+                    msg = msg.encode('utf8')               
+                net6.sendto(msg, addr[4])
     else:
         net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
         for idx in util.list_interface_indexes():
             net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
                             idx)
             try:
-                net6.sendto(smsg.format('[{0}]'.format(mcastv6addr), service
-                                        ), (mcastv6addr, 1900, 0, 0))
+                msg = smsg.format('[{0}]'.format(mcastv6addr), service)
+                if not isinstance(msg, bytes):
+                    msg = msg.encode('utf8')
+                net6.sendto(msg, (mcastv6addr, 1900, 0, 0))
             except socket.error:
                 # ignore interfaces without ipv6 multicast causing error
                 pass
@@ -185,9 +229,14 @@ def _find_service(service, target):
             bcast = i4['broadcast']
             net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
                             socket.inet_aton(addr))
-            net4.sendto(smsg.format(mcastv4addr, service),
-                        (mcastv4addr, 1900))
-            net4.sendto(smsg.format(bcast, service), (bcast, 1900))
+            msg = smsg.format(mcastv4addr, service)
+            if not isinstance(msg, bytes):
+                msg = msg.encode('utf8')
+            net4.sendto(msg, (mcastv4addr, 1900))
+            msg = smsg.format(bcast, service)
+            if not isinstance(msg, bytes):
+                msg = msg.encode('utf8')
+            net4.sendto(msg, (bcast, 1900))
     # SSDP by spec encourages responses to spread out over a 3 second interval
     # hence we must be a bit more patient
     deadline = util.monotonic_time() + 4

confluent_server/confluent/httpapi.py

@@ -30,6 +30,7 @@ import confluent.log as log
 import confluent.messages
 import confluent.core as pluginapi
 import confluent.asynchttp
+import confluent.selfservice as selfservice
 import confluent.shellserver as shellserver
 import confluent.tlvdata
 import confluent.util as util
@@ -411,12 +412,16 @@ def resourcehandler_backend(env, start_response):
                ('X-Permitted-Cross-Domain-Policies', 'none')]
     reqbody = None
     reqtype = None
+    if env.get('PATH_INFO', '').startswith('/self/'):
+        for res in selfservice.handle_request(env, start_response):
+            yield res
+        return
     if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
         reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
         reqtype = env['CONTENT_TYPE']
     operation = opmap[env['REQUEST_METHOD']]
     querydict = _get_query_dict(env, reqbody, reqtype)
-    if 'restexplorerop' in querydict:
+    if operation != 'retrieve' and 'restexplorerop' in querydict:
         operation = querydict['restexplorerop']
         del querydict['restexplorerop']
     authorized = _authorize_request(env, operation)

confluent_server/confluent/main.py

@@ -34,6 +34,7 @@ import confluent.core as confluentcore
 import confluent.httpapi as httpapi
 import confluent.log as log
 import confluent.collective.manager as collective
+import confluent.discovery.protocols.pxe as pxe
 try:
     import confluent.sockapi as sockapi
 except ImportError:
@@ -262,6 +263,7 @@ def run(args):
     webservice = httpapi.HttpApi(http_bind_host, http_bind_port)
     webservice.start()
     disco.start_detection()
+    pxe.start_proxydhcp()
     try:
         sockservice = sockapi.SockApi(sock_bind_host, sock_bind_port)
         sockservice.start()

confluent_server/confluent/messages.py

@@ -674,6 +674,8 @@ class InputAttributes(ConfluentMessage):
         if nodes is None:
             self.attribs = inputdata
             for attrib in self.attribs:
+                if not cfm.attrib_supports_expression(attrib):
+                    continue
                 if type(self.attribs[attrib]) in (bytes, unicode):
                     try:
                         # ok, try to use format against the string
@@ -700,7 +702,7 @@ class InputAttributes(ConfluentMessage):
             return {}
         nodeattr = deepcopy(self.nodeattribs[node])
         for attr in nodeattr:
-            if type(nodeattr[attr]) in (bytes, unicode):
+            if type(nodeattr[attr]) in (bytes, unicode) and cfm.attrib_supports_expression(attr):
                 try:
                     # as above, use format() to see if string follows
                     # expression, store value back in case of escapes
@@ -1798,8 +1800,12 @@ class CryptedAttributes(Attributes):
         nkv = {}
         for key in kv:
             nkv[key] = {'isset': False}
+            if 'hashvalue' in kv[key]:
+                targkey = 'hashvalue'
+            else:
+                targkey = 'cryptvalue'
             try:
-                if kv[key] is not None and kv[key]['cryptvalue'] != '':
+                if kv[key] is not None and kv[key][targkey] != '':
                     nkv[key] = {'isset': True}
                     nkv[key]['inheritedfrom'] = kv[key]['inheritedfrom']
             except KeyError:

confluent_server/confluent/netutil.py

@@ -18,9 +18,11 @@
 
 import confluent.exceptions as exc
 import codecs
+import netifaces
 import struct
 import eventlet.green.socket as socket
 import eventlet.support.greendns
+import os
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
 
@@ -38,6 +40,10 @@ def cidr_to_mask(cidr):
         socket.AF_INET, struct.pack('!I', (2**32 - 1) ^ (2**(32 - cidr) - 1)))
 
 def ip_on_same_subnet(first, second, prefix):
+    if first.startswith('::ffff:') and '.' in first:
+        first = first.replace('::ffff:', '')
+    if second.startswith('::ffff:') and '.' in second:
+        second = second.replace('::ffff:', '')
     addrinf = socket.getaddrinfo(first, None, 0, socket.SOCK_STREAM)[0]
     fam = addrinf[0]
     ip = socket.inet_pton(fam, addrinf[-1][0])
@@ -45,7 +51,8 @@ def ip_on_same_subnet(first, second, prefix):
     addrinf = socket.getaddrinfo(second, None, 0, socket.SOCK_STREAM)[0]
     if fam != addrinf[0]:
         return False
-    oip = socket.inet_pton(fam, addrinf[-1][0])
+    txtaddr = addrinf[-1][0].split('%')[0]
+    oip = socket.inet_pton(fam, txtaddr)
     oip = int(codecs.encode(bytes(oip), 'hex'), 16)
     if fam == socket.AF_INET:
         addrlen = 32
@@ -57,6 +64,64 @@ def ip_on_same_subnet(first, second, prefix):
     return ip & mask == oip & mask
 
 
+def address_is_local(address):
+    for iface in netifaces.interfaces():
+        for i4 in netifaces.ifaddresses(iface).get(2, []):
+            cidr = mask_to_cidr(i4['netmask'])
+            if ip_on_same_subnet(i4['addr'], address, cidr):
+                return True
+        for i6 in netifaces.ifaddresses(iface).get(10, []):
+            cidr = int(i6['netmask'].split('/')[1])
+            laddr = i6['addr'].split('%')[0]
+            if ip_on_same_subnet(laddr, address, cidr):
+                return True
+    return False
+
+
+_idxtoifnamemap = {}
+def _rebuildidxmap():
+    _idxtoifnamemap.clear()
+    for iname in os.listdir('/sys/class/net'):
+        ci = int(open('/sys/class/net/{0}/ifindex'.format(iname)).read())
+        _idxtoifnamemap[ci] = iname
+
+
+def myiptonets(svrip):
+    fam = netifaces.AF_INET
+    if ':' in svrip:
+        fam = netifaces.AF_INET6
+    relevantnic = None
+    for iface in netifaces.interfaces():
+        for addr in netifaces.ifaddresses(iface).get(fam, []):
+            addr = addr.get('addr', '')
+            addr = addr.split('%')[0]
+            if addresses_match(addr, svrip):
+                relevantnic = iface
+                break
+        else:
+            continue
+        break
+    return inametonets(relevantnic)
+
+
+def idxtonets(ifidx):
+    _rebuildidxmap()
+    iname = _idxtoifnamemap.get(ifidx, None)
+    return inametonets(iname)
+
+def inametonets(iname):
+    addrs = netifaces.ifaddresses(iname)
+    try:
+        addrs = addrs[netifaces.AF_INET]
+    except KeyError:
+        return
+    for addr in addrs:
+        ip = struct.unpack('!I', socket.inet_aton(addr['addr']))[0]
+        mask = struct.unpack('!I', socket.inet_aton(addr['netmask']))[0]
+        net = ip & mask
+        net = socket.inet_ntoa(struct.pack('!I', net))
+        yield (net, mask_to_cidr(addr['netmask']), addr['addr'])
+
 # TODO(jjohnson2): have a method to arbitrate setting methods, to aid
 # in correct matching of net.* based on parameters, mainly for pxe
 # The scheme for pxe:
@@ -66,19 +131,21 @@ def ip_on_same_subnet(first, second, prefix):
 # that mac address
 # the ip as reported by recvmsg to match the subnet of that net.* interface
 # if switch and port available, that should match.
-def get_nic_config(configmanager, node, ip=None, mac=None):
+def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
+                   serverip=None):
     """Fetch network configuration parameters for a nic
-    
+
     For a given node and interface, find and retrieve the pertinent network
     configuration data.  The desired configuration can be searched
     either by ip or by mac.
-    
-    :param configmanager: The relevant confluent.config.ConfigManager 
+
+    :param configmanager: The relevant confluent.config.ConfigManager
         instance.
     :param node:  The name of the node
     :param ip:  An IP address on the intended subnet
     :param mac: The mac address of the interface
-    
+    :param ifidx: The local index relevant to the network.
+
     :returns: A dict of parameters, 'ipv4_gateway', ....
     """
     # ip parameter *could* be the result of recvmsg with cmsg to tell
@@ -87,15 +154,93 @@ def get_nic_config(configmanager, node, ip=None, mac=None):
     # join a bond/bridge, vlan configs, etc.
     # also other nic criteria, physical location, driver and index...
     nodenetattribs = configmanager.get_node_attributes(
-        node, 'net*.ipv4_gateway').get(node, {})
+        node, 'net*').get(node, {})
+    cfgbyname = {}
+    for attrib in nodenetattribs:
+        segs = attrib.split('.')
+        if len(segs) == 2:
+            name = None
+        else:
+            name = segs[1]
+        if name not in cfgbyname:
+            cfgbyname[name] = {}
+        cfgbyname[name][segs[-1]] = nodenetattribs[attrib].get('value',
+                                                                None)
     cfgdata = {
         'ipv4_gateway': None,
+        'ipv4_address': None,
+        'ipv4_method': None,
         'prefix': None,
     }
+    nets = None
+    needsvrip = False
+    if ifidx is not None:
+        dhcprequested = False
+        nets = list(idxtonets(ifidx))
+        if not nets:
+            cfgdata['ipv4_broken'] = True
+    if serverip is not None:
+        needsvrip = True
+        dhcprequested = False
+        nets = list(myiptonets(serverip))
+    if nets is not None:
+        candgws = []
+        candsrvs = []
+        for net in nets:
+            net, prefix, svrip = net
+            candsrvs.append(svrip)
+            cfgdata['deploy_server'] = svrip
+            for candidate in cfgbyname:
+                if cfgbyname[candidate].get('ipv4_method', None) == 'dhcp':
+                    dhcprequested = True
+                    continue
+                candip = cfgbyname[candidate].get('ipv4_address', None)
+                if candip and '/' in candip:
+                    candip, candprefix = candip.split('/')
+                    if int(candprefix) != prefix:
+                        continue
+                candgw = cfgbyname[candidate].get('ipv4_gateway', None)
+                if candip:
+                    if ip_on_same_subnet(net, candip, prefix):
+                        cfgdata['ipv4_address'] = candip
+                        cfgdata['ipv4_method'] = 'static'
+                        cfgdata['ipv4_gateway'] = cfgbyname[candidate].get(
+                            'ipv4_gateway', None)
+                        cfgdata['prefix'] = prefix
+                        return cfgdata
+                elif candgw:
+                    if ip_on_same_subnet(net, candgw, prefix):
+                        candgws.append(candgw)
+        if dhcprequested:
+            return cfgdata
+        ipbynodename = None
+        try:
+            ipbynodename = socket.getaddrinfo(
+                node, 0, socket.AF_INET, socket.SOCK_DGRAM)[0][-1][0]
+        except Exception:
+            return cfgdata
+        for net in nets:
+            net, prefix, svrip = net
+            if ip_on_same_subnet(net, ipbynodename, prefix):
+                cfgdata['ipv4_address'] = ipbynodename
+                cfgdata['ipv4_method'] = 'static'
+                cfgdata['prefix'] = prefix
+                break
+        for svr in candsrvs:
+            if ip_on_same_subnet(svr, ipbynodename, prefix):
+                cfgdata['deploy_server'] = svr
+                break
+        for gw in candgws:
+            if ip_on_same_subnet(gw, ipbynodename, prefix):
+                cfgdata['ipv4_gateway'] = gw
+                break
+        return cfgdata
     if ip is not None:
         prefixlen = get_prefix_len_for_ip(ip)
         cfgdata['prefix'] = prefixlen
         for setting in nodenetattribs:
+            if 'ipv4_gateway' not in setting:
+                continue
             gw = nodenetattribs[setting].get('value', None)
             if gw is None or not gw:
                 continue

confluent_server/confluent/osimage.py

@@ -0,0 +1,457 @@
+#!/usr/bin/python
+import confluent.messages as msg
+import eventlet
+import eventlet.green.select as select
+import eventlet.green.subprocess as subprocess
+import glob
+import logging
+logging.getLogger('libarchive').addHandler(logging.NullHandler())
+import libarchive
+import hashlib
+import os
+import shutil
+import sys
+import time
+import yaml
+
+COPY = 1
+EXTRACT = 2
+READFILES = set([
+    'README.diskdefines',
+    'media.1/products',
+    'media.2/products',
+    '.discinfo',
+])
+
+HEADERSUMS = set([b'\x85\xeddW\x86\xc5\xbdhx\xbe\x81\x18X\x1e\xb4O\x14\x9d\x11\xb7C8\x9b\x97R\x0c-\xb8Ht\xcb\xb3'])
+HASHPRINTS = {
+    '69d5f1c5e4474d70b0fb5374bfcb29bf57ba828ff00a55237cd757e61ed71048': {'name': 'cumulus-broadcom-amd64-4.0.0', 'method': COPY},
+}
+
+from ctypes import byref, c_longlong, c_size_t, c_void_p
+
+from libarchive.ffi import (
+    write_disk_new, write_disk_set_options, write_free, write_header,
+    read_data_block, write_data_block, write_finish_entry, ARCHIVE_EOF
+)
+
+def relax_umask():
+    os.umask(0o22)
+
+def update_boot(profiledir):
+    profile = {}
+    if profiledir.endswith('/'):
+        profiledir = profiledir[:-1]
+    profname = os.path.basename(profiledir)
+    with open('{0}/profile.yaml'.format(profiledir)) as profileinfo:
+        profile = yaml.safe_load(profileinfo)
+    label = profile.get('label', profname)
+    kernelargs = profile.get('kernelargs', '')
+    grubcfg = "set timeout=5\nmenuentry '"
+    grubcfg += label
+    grubcfg += "' {\n    linuxefi /kernel " + kernelargs + "\n"
+    initrds = []
+    for initramfs in glob.glob(profiledir + '/boot/initramfs/*.cpio'):
+        initramfs = os.path.basename(initramfs)
+        initrds.append(initramfs)
+    for initramfs in os.listdir(profiledir + '/boot/initramfs'):
+        if initramfs not in initrds:
+            initrds.append(initramfs)
+    grubcfg += "    initrdefi "
+    for initramfs in initrds:
+        grubcfg += " /initramfs/{0}".format(initramfs)
+    grubcfg += "\n}\n"
+    with open(profiledir + '/boot/efi/boot/grub.cfg', 'w') as grubout:
+        grubout.write(grubcfg)
+    ipxeargs = kernelargs
+    for initramfs in initrds:
+        ipxeargs += " initrd=" + initramfs
+    oum = os.umask(0o22)
+    ipout = os.open(profiledir + '/boot/boot.ipxe', os.O_WRONLY|os.O_CREAT, 0o644)
+    ipxeout = os.fdopen(ipout, 'w')
+    try:
+        os.umask(oum)
+        ipxeout.write('#!ipxe\n')
+        ipxeout.write('imgfetch kernel ' + ipxeargs + '\n')
+        for initramfs in initrds:
+            ipxeout.write('imgfetch initramfs/{0}\n'.format(initramfs))
+        ipxeout.write('imgload kernel\nimgexec kernel\n')
+    finally:
+        ipxeout.close()
+    subprocess.check_call(
+        ['/opt/confluent/bin/dir2img', '{0}/boot'.format(profiledir),
+         '{0}/boot.img'.format(profiledir)], preexec_fn=relax_umask)
+
+
+def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist=None):
+    """Extracts the given archive entries into the current directory.
+    """
+    buff, size, offset = c_void_p(), c_size_t(), c_longlong()
+    buff_p, size_p, offset_p = byref(buff), byref(size), byref(offset)
+    sizedone = 0
+    printat = 0
+    with libarchive.extract.new_archive_write_disk(flags) as write_p:
+        for entry in entries:
+            if str(entry).endswith('TRANS.TBL'):
+                continue
+            if extractlist and str(entry) not in extractlist:
+                continue
+            write_header(write_p, entry._entry_p)
+            read_p = entry._archive_p
+            while 1:
+                r = read_data_block(read_p, buff_p, size_p, offset_p)
+                sizedone += size.value
+                if callback and time.time() > printat:
+                    callback({'progress': float(sizedone) / float(totalsize)})
+                    printat = time.time() + 0.5
+                if r == ARCHIVE_EOF:
+                    break
+                write_data_block(write_p, buff, size, offset)
+            write_finish_entry(write_p)
+    if callback:
+        callback({'progress': float(sizedone) / float(totalsize)})
+
+
+def extract_file(filepath, flags=0, callback=lambda x: None, imginfo=(), extractlist=None):
+    """Extracts an archive from a file into the current directory."""
+    totalsize = 0
+    for img in imginfo:
+        if not imginfo[img]:
+            continue
+        totalsize += imginfo[img]
+    with libarchive.file_reader(filepath) as archive:
+        extract_entries(archive, flags, callback, totalsize, extractlist)
+
+
+def check_centos(isoinfo):
+    ver = None
+    arch = None
+    cat = None
+    for entry in isoinfo[0]:
+        if 'centos-release-7' in entry:
+            dotsplit = entry.split('.')
+            arch = dotsplit[-2]
+            ver = dotsplit[0].split('release-')[-1].replace('-', '.')
+            cat = 'el7'
+            break
+        elif 'centos-release-8' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            cat = 'el8'
+            break
+    else:
+        return None
+    return {'name': 'centos-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': cat}
+
+
+def check_ubuntu(isoinfo):
+    if 'README.diskdefines' not in isoinfo[1]:
+        return None
+    arch = None
+    variant = None
+    ver = None
+    diskdefs = isoinfo[1]['README.diskdefines']
+    for info in diskdefs.split(b'\n'):
+        if not info:
+            continue
+        _, key, val = info.split(b' ', 2)
+        val = val.strip()
+        if key == b'ARCH':
+            arch = val
+            if arch == b'amd64':
+                arch = b'x86_64'
+        elif key == b'DISKNAME':
+            variant, ver, _ = val.split(b' ', 2)
+            if variant != b'Ubuntu-Server':
+                return None
+    if variant:
+        if not isinstance(ver, str):
+            ver = ver.decode('utf8')
+        if not isinstance(arch, str):
+            arch = arch.decode('utf8')
+        major = '.'.join(ver.split('.', 2)[:2])
+        return {'name': 'ubuntu-{0}-{1}'.format(ver, arch),
+                'method': EXTRACT|COPY,
+                'extractlist': ['casper/vmlinuz', 'casper/initrd',
+                'EFI/BOOT/BOOTx64.EFI', 'EFI/BOOT/grubx64.efi'
+                ],
+                'copyto': 'install.iso',
+                'category': 'ubuntu{0}'.format(major)}
+
+
+def check_sles(isoinfo):
+    ver = None
+    arch = 'x86_64'
+    disk = None
+    distro = ''
+    if 'media.1/products' in isoinfo[1]:
+        medianame = 'media.1/products'
+    elif 'media.2/products' in isoinfo[1]:
+        medianame = 'media.2/products'
+    else:
+        return None
+    prodinfo = isoinfo[1][medianame]
+    if not isinstance(prodinfo, str):
+        prodinfo = prodinfo.decode('utf8')
+    prodinfo = prodinfo.split('\n')
+    hline = prodinfo[0].split(' ')
+    ver = hline[-1].split('-')[0]
+    major = ver.split('.', 2)[0]
+    if hline[-1].startswith('15'):
+        if hline[1] == 'openSUSE-Leap':
+            distro = 'opensuse_leap'
+        else:
+            distro = 'sle'
+        if hline[0] == '/' or 'boot' in isoinfo[0]:
+            disk = '1'
+        elif hline[0].startswith('/Module'):
+            disk = '2'
+    elif hline[-1].startswith('12'):
+        if 'SLES' in hline[1]:
+            distro = 'sles'
+        if '.1' in medianame:
+            disk = '1'
+        elif '.2' in medianame:
+            disk = '2'
+    if disk and distro:
+        return {'name': '{0}-{1}-{2}'.format(distro, ver, arch),
+                'method': EXTRACT, 'subname': disk,
+                'category': 'suse{0}'.format(major)}
+    return None
+
+
+def check_rhel(isoinfo):
+    ver = None
+    arch = None
+    for entry in isoinfo[0]:
+        if 'redhat-release-7' in entry:
+            dotsplit = entry.split('.')
+            arch = dotsplit[-2]
+            ver = dotsplit[0].split('release-')[-1].replace('-', '.')
+            break
+        elif 'redhat-release-8' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            break
+    else:
+        return None
+    major = ver.split('.', 1)[0]
+    return {'name': 'rhel-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': 'el{0}'.format(major)}
+
+
+def scan_iso(filename):
+    filesizes = {}
+    filecontents = {}
+    with libarchive.file_reader(filename) as reader:
+        for ent in reader:
+            if str(ent).endswith('TRANS.TBL'):
+                continue
+            eventlet.sleep(0)
+            filesizes[str(ent)] = ent.size
+            if str(ent) in READFILES:
+                filecontents[str(ent)] = b''
+                for block in ent.get_blocks():
+                    filecontents[str(ent)] += bytes(block)
+    return filesizes, filecontents
+
+
+def fingerprint(filename):
+    with open(filename, 'rb') as archive:
+        header = archive.read(32768)
+        archive.seek(32769)
+        if archive.read(6) == b'CD001\x01':
+            # ISO image
+            isoinfo = scan_iso(filename)
+            name = None
+            for fun in globals():
+                if fun.startswith('check_'):
+                    name = globals()[fun](isoinfo)
+                    if name:
+                        return name, isoinfo[0]
+            return None
+        else:
+            sum = hashlib.sha256(header)
+            if sum.digest() in HEADERSUMS:
+                archive.seek(32768)
+                chunk = archive.read(32768)
+                while chunk:
+                    sum.update(chunk)
+                    chunk = archive.read(32768)
+                imginfo = HASHPRINTS.get(sum.hexdigest(), None)
+                if imginfo:
+                    return imginfo, None
+
+
+def import_image(filename, callback, backend=False):
+    identity = fingerprint(filename)
+    if not identity:
+        return -1
+    identity, imginfo = identity
+    targpath = identity['name']
+    if identity.get('subname', None):
+        targpath += '/' + identity['subname']
+    targpath = '/var/lib/confluent/distributions/' + targpath
+    os.makedirs(targpath, 0o755)
+    filename = os.path.abspath(filename)
+    os.chdir(targpath)
+    if not backend:
+        print('Importing OS to ' + targpath + ':')
+    printit({'progress': 0.0})
+    if EXTRACT & identity['method']:
+        extract_file(filename, callback=callback, imginfo=imginfo, extractlist=identity.get('extractlist', None))
+    if COPY & identity['method']:
+        basename = identity.get('copyto', os.path.basename(filename))
+        targpath = os.path.join(targpath, basename)
+        shutil.copyfile(filename, targpath)
+    printit({'progress': 1.0})
+    sys.stdout.write('\n')
+
+def printit(info):
+    sys.stdout.write('     \r{:.2f}%'.format(100 * info['progress']))
+    sys.stdout.flush()
+
+
+def list_distros():
+    return os.listdir('/var/lib/confluent/distributions')
+
+def list_profiles():
+    return os.listdir('/var/lib/confluent/public/os/')
+
+def get_profile_label(profile):
+    with open('/var/lib/confluent/public/os/{0}/profile.yaml') as metadata:
+        prof = yaml.safe_load(metadata)
+    return prof.get('label', profile)
+
+importing = {}
+class MediaImporter(object):
+
+    def __init__(self, media):
+        self.worker = None
+        self.profiles = []
+        identity = fingerprint(media)
+        self.percent = 0.0
+        identity, _ = identity
+        self.phase = 'copying'
+        if not identity:
+            raise Exception('Unrecognized OS Media')
+        if 'subname' in identity:
+            importkey = '{0}-{1}'.format(identity['name'], identity['subname'])
+        else:
+            importkey = identity['name']
+        if importkey in importing:
+            raise Exception('Media import already in progress for this media')
+        self.importkey = importkey
+        importing[importkey] = self
+        self.importkey = importkey
+        self.osname = identity['name']
+        self.oscategory = identity.get('category', None)
+        targpath = identity['name']
+        self.distpath = '/var/lib/confluent/distributions/' + targpath
+        if identity.get('subname', None):
+            targpath += '/' + identity['subname']
+        self.targpath = '/var/lib/confluent/distributions/' + targpath
+        if os.path.exists(self.targpath):
+            raise Exception('{0} already exists'.format(self.targpath))
+        self.filename = os.path.abspath(media)
+        self.importer = eventlet.spawn(self.importmedia)
+
+    def stop(self):
+        if self.worker and self.worker.poll() is None:
+            self.worker.kill()
+
+    @property
+    def progress(self):
+        return {'phase': self.phase, 'progress': self.percent, 'profiles': self.profiles}
+
+    def importmedia(self):
+        os.environ['PYTHONPATH'] = ':'.join(sys.path)
+        with open(os.devnull, 'w') as devnull:
+            self.worker = subprocess.Popen(
+                [sys.executable, __file__, self.filename, '-b'],
+                stdin=devnull, stdout=subprocess.PIPE)
+        wkr = self.worker
+        currline = b''
+        while wkr.poll() is None:
+            currline += wkr.stdout.read(1)
+            if b'\r' in currline:
+                val = currline.split(b'%')[0].strip()
+                if val:
+                    self.percent = float(val)
+                currline = b''
+        a = wkr.stdout.read(1)
+        while a:
+            currline += a
+            if b'\r' in currline:
+                val = currline.split(b'%')[0].strip()
+                if val:
+                    self.percent = float(val)
+            currline = b''
+            a = wkr.stdout.read(1)
+        bootupdates = []
+        if self.oscategory:
+            defprofile = '/opt/confluent/lib/osdeploy/{0}'.format(
+                self.oscategory)
+            osd, osversion, arch = self.osname.split('-')
+            for prof in os.listdir('{0}/profiles'.format(defprofile)):
+                srcname = '{0}/profiles/{1}'.format(defprofile, prof)
+                profname = '{0}-{1}'.format(self.osname, prof)
+                dirname = '/var/lib/confluent/public/os/{0}'.format(profname)
+                if os.path.exists(dirname):
+                    continue
+                oumask = os.umask(0o22)
+                shutil.copytree(srcname, dirname)
+                profdata = None
+                try:
+                    os.makedirs('{0}/boot/initramfs'.format(dirname), 0o755)
+                except OSError as e:
+                    if e.errno != 17:
+                        raise
+                finally:
+                    os.umask(oumask)
+                with open('{0}/profile.yaml'.format(dirname)) as yin:
+                    profdata = yin.read()
+                    profdata = profdata.replace('%%DISTRO%%', osd)
+                    profdata = profdata.replace('%%VERSION%%', osversion)
+                    profdata = profdata.replace('%%ARCH%%', arch)
+                    profdata = profdata.replace('%%PROFILE%%', profname)
+                if profdata:
+                    with open('{0}/profile.yaml'.format(dirname), 'w') as yout:
+                        yout.write(profdata)
+                for initrd in os.listdir('{0}/initramfs'.format(defprofile)):
+                    fullpath = '{0}/initramfs/{1}'.format(defprofile, initrd)
+                    os.symlink(fullpath, '{0}/boot/initramfs/{1}'.format(dirname, initrd))
+                os.symlink(
+                    '/var/lib/confluent/public/site/initramfs.cpio',
+                    '{0}/boot/initramfs/site.cpio'.format(dirname))
+                os.symlink(self.distpath, '{0}/distribution'.format(dirname))
+                subprocess.check_call(
+                    ['sh', '{0}/initprofile.sh'.format(dirname),
+                    self.targpath, dirname])
+                bootupdates.append(eventlet.spawn(update_boot, dirname))
+                self.profiles.append(profname)
+        for upd in bootupdates:
+            upd.wait()
+        self.phase = 'complete'
+        self.percent = 100.0
+
+
+def list_importing():
+    return [msg.ChildCollection(x) for x in importing]
+
+
+def remove_importing(importkey):
+    importing[importkey].stop()
+    del importing[importkey]
+    yield msg.DeletedResource('deployment/importing/{0}'.format(importkey))
+
+
+def get_importing_status(importkey):
+    yield msg.KeyValueData(importing[importkey].progress)
+
+
+if __name__ == '__main__':
+    os.umask(0o022)
+    if len(sys.argv) > 2:
+        sys.exit(import_image(sys.argv[1], callback=printit, backend=True))
+    else:
+        sys.exit(import_image(sys.argv[1], callback=printit))

confluent_server/confluent/plugins/configuration/attributes.py

@@ -57,7 +57,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                 val['desc'] = 'The noderange this group is expanded ' \
                     'to when used in noderange, exclusive with static ' \
                     'nodes'
-            if attribute.startswith('secret.'):
+            if attribute.startswith('secret.') or attribute.startswith('crypted.'):
                 yield msg.CryptedAttributes(
                     kv={attribute: val},
                     desc=allattributes.node[attribute]['description'])
@@ -117,7 +117,7 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                     val = []
                 else:  # no setting, provide a blank
                     val = {'value': None}
-                if attribute.startswith('secret.'):
+                if attribute.startswith('secret.') or attribute.startswith('crypted.'):
                     yield msg.CryptedAttributes(
                         node, {attribute: val},
                         allattributes.node.get(
@@ -142,7 +142,7 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                     desc = ''
                 if 'value' in currattr or 'expression' in currattr:
                     yield msg.Attributes(node, {attribute: currattr}, desc)
-                elif 'cryptvalue' in currattr:
+                elif 'cryptvalue' in currattr or 'hashvalue' in currattr:
                     yield msg.CryptedAttributes(
                         node, {attribute: currattr}, desc)
                 elif isinstance(currattr, list):

confluent_server/confluent/selfservice.py

@@ -0,0 +1,170 @@
+import confluent.config.configmanager as configmanager
+import confluent.collective.manager as collective
+import confluent.netutil as netutil
+import confluent.sshutil as sshutil
+import confluent.util as util
+import eventlet.green.subprocess as subprocess
+import crypt
+import json
+import time
+import yaml
+
+currtz = None
+keymap = 'us'
+currlocale = 'en_US.UTF-8'
+currtzvintage = None
+
+
+def yamldump(input):
+    return yaml.safe_dump(input, default_flow_style=False)
+
+
+def handle_request(env, start_response):
+    global currtz
+    global keymap
+    global currlocale
+    global currtzvintage
+    nodename = env.get('HTTP_CONFLUENT_NODENAME', None)
+    apikey = env.get('HTTP_CONFLUENT_APIKEY', None)
+    if not (nodename and apikey):
+        start_response('401 Unauthorized', [])
+        yield 'Unauthorized'
+        return
+    cfg = configmanager.ConfigManager(None)
+    eak = cfg.get_node_attributes(nodename, 'crypted.selfapikey').get(
+        nodename, {}).get('crypted.selfapikey', {}).get('hashvalue', None)
+    if not eak:
+        start_response('401 Unauthorized', [])
+        yield 'Unauthorized'
+        return
+    salt = '$'.join(eak.split('$', 3)[:-1]) + '$'
+    if crypt.crypt(apikey, salt) != eak:
+        start_response('401 Unauthorized', [])
+        yield 'Unauthorized'
+        return
+    retype = env.get('HTTP_ACCEPT', 'application/yaml')
+    isgeneric = False
+    if retype == '*/*':
+        isgeneric = True
+        retype = 'application/yaml'
+    if retype == 'application/yaml':
+        dumper = yamldump
+    elif retype == 'application/json':
+        dumper = json.dumps
+    else:
+        start_response('406 Not supported', [])
+        yield 'Unsupported content type in ACCEPT: ' + retype
+        return
+    if env['REQUEST_METHOD'] not in ('HEAD', 'GET') and 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
+        reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
+    if env['PATH_INFO'] == '/self/deploycfg':
+        myip = env.get('HTTP_X_FORWARDED_HOST', None)
+        myip = myip.replace('[', '').replace(']', '')
+        ncfg = netutil.get_nic_config(cfg, nodename, serverip=myip)
+        if ncfg['prefix']:
+            ncfg['ipv4_netmask'] = netutil.cidr_to_mask(ncfg['prefix'])
+        deployinfo = cfg.get_node_attributes(
+            nodename, ('deployment.*', 'console.method', 'crypted.rootpassword',
+                       'dns.*'))
+        deployinfo = deployinfo.get(nodename, {})
+        profile = deployinfo.get(
+            'deployment.pendingprofile', {}).get('value', '')
+        ncfg['profile'] = profile
+        protocol = deployinfo.get('deployment.useinsecureprotocols', {}).get(
+            'value', 'never')
+        ncfg['textconsole'] = bool(deployinfo.get(
+                                  'console.method', {}).get('value', None))
+        if protocol == 'always':
+            ncfg['protocol'] = 'http'
+        else:
+            ncfg['protocol'] = 'https'
+        ncfg['rootpassword'] = deployinfo.get('crypted.rootpassword', {}).get(
+            'hashvalue', None)
+        if currtzvintage and currtzvintage > (time.time() - 30.0):
+            ncfg['timezone'] = currtz
+        else:
+            langinfo = subprocess.check_output(
+                ['localectl', 'status']).split(b'\n')
+            for line in langinfo:
+                line = line.strip()
+                if line.startswith(b'System Locale:'):
+                    ccurrlocale = line.split(b'=')[-1]
+                    if not ccurrlocale:
+                        continue
+                    if not isinstance(ccurrlocale, str):
+                        ccurrlocale = ccurrlocale.decode('utf8')
+                    if ccurrlocale == 'n/a':
+                        continue
+                    currlocale = ccurrlocale
+                elif line.startswith(b'VC Keymap:'):
+                    ckeymap = line.split(b':')[-1]
+                    ckeymap = ckeymap.strip()
+                    if not ckeymap:
+                        continue
+                    if not isinstance(ckeymap, str):
+                        ckeymap = ckeymap.decode('utf8')
+                    if ckeymap == 'n/a':
+                        continue
+                    keymap = ckeymap
+            tdc = subprocess.check_output(['timedatectl']).split(b'\n')
+            for ent in tdc:
+                ent = ent.strip()
+                if ent.startswith(b'Time zone:'):
+                    currtz = ent.split(b': ', 1)[1].split(b'(', 1)[0].strip()
+                    if not isinstance(currtz, str):
+                        currtz = currtz.decode('utf8')
+                    currtzvintage = time.time()
+                    ncfg['timezone'] = currtz
+                    break
+        ncfg['locale'] = currlocale
+        ncfg['keymap'] = keymap
+        ncfg['nameservers'] = []
+        for dns in deployinfo.get(
+                'dns.servers', {}).get('value', '').split(','):
+            ncfg['nameservers'].append(dns)
+        dnsdomain = deployinfo.get('dns.domain', {}).get('value', None)
+        ncfg['dnsdomain'] = dnsdomain
+        start_response('200 OK', (('Content-Type', retype),))
+        yield dumper(ncfg)
+    elif env['PATH_INFO'] == '/self/sshcert':
+        if not sshutil.ca_exists():
+            start_response('500 Unconfigured', ())
+            yield 'CA is not configured on this system (run ...)'
+            return
+        cert = sshutil.sign_host_key(reqbody, nodename)
+        start_response('200 OK', (('Content-Type', 'text/plain'),))
+        yield cert
+    elif env['PATH_INFO'] == '/self/nodelist':
+        nodes = set(cfg.list_nodes())
+        for mgr in configmanager.list_collective():
+            nodes.add(mgr)
+        nodes.add(collective.get_myname())
+        if isgeneric:
+            start_response('200 OK', (('Content-Type', 'text/plain'),))
+            for node in util.natural_sort(nodes):
+                yield node + '\n'
+        else:
+            start_response('200 OK', (('Content-Type', retype),))
+            yield dumper(sorted(nodes))
+    elif env['PATH_INFO'] == '/self/updatestatus':
+        update = yaml.safe_load(reqbody)
+        if update['status'] != 'complete':
+            raise Exception('Unknown update status request')
+        currattr = cfg.get_node_attributes(nodename, 'deployment.*').get(
+            nodename, {})
+        pending = currattr.get('deployment.pendingprofile', {}).get('value', '')
+        updates = {}
+        if pending:
+            updates['deployment.pendingprofile'] = {'value': ''}
+            currprof = currattr.get('deployment.profile', {}).get('value', '')
+            if currprof != pending:
+                updates['deployment.profile'] = {'value': pending}
+            cfg.set_node_attributes({nodename: updates})
+            start_response('200 OK', (('Content-Type', 'text/plain'),))
+            yield 'OK'
+        else:
+            start_response('500 Error', (('Content-Type', 'text/plain'),))
+            yield 'No pending profile detected, unable to accept status update'
+    else:
+        start_response('404 Not Found', ())
+        yield 'Not found'

confluent_server/confluent/sockapi.py

@@ -37,6 +37,7 @@ import eventlet.green.ssl as ssl
 import eventlet
 
 import confluent.auth as auth
+import confluent.credserver as credserver
 import confluent.tlvdata as tlvdata
 import confluent.consoleserver as consoleserver
 import confluent.config.configmanager as configmanager
@@ -352,9 +353,13 @@ def _tlshandler(bind_host, bind_port):
     # Enable TCP_FASTOPEN
     plainsocket.setsockopt(socket.SOL_TCP, 23, 5)
     plainsocket.listen(5)
+    cs = credserver.CredServer()
     while (1):  # TODO: exithook
         cnn, addr = plainsocket.accept()
-        eventlet.spawn_n(_tlsstartup, cnn)
+        if addr[1] < 1000:
+            eventlet.spawn_n(cs.handle_client, cnn, addr)
+        else:
+            eventlet.spawn_n(_tlsstartup, cnn)
 
 
 if ffi:
@@ -473,9 +478,10 @@ class SockApi(object):
     def watch_for_cert(self):
         libc = ctypes.CDLL(ctypes.util.find_library('c'))
         watcher = libc.inotify_init()
-        if libc.inotify_add_watch(watcher, '/etc/confluent/', 0x100) > -1:
+        if libc.inotify_add_watch(watcher, b'/etc/confluent/', 0x100) > -1:
             while True:
                 select.select((watcher,), (), (), 86400)
+                os.read(watcher, 1024)
                 if self.should_run_remoteapi():
                     os.close(watcher)
                     self.start_remoteapi()

confluent_server/confluent/sshutil.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python
+
+import confluent.collective.manager as collective
+import eventlet.green.subprocess as subprocess
+import glob
+import os
+import shutil
+import tempfile
+
+def normalize_uid():
+    curruid = os.geteuid()
+    neededuid = os.stat('/etc/confluent').st_uid
+    if curruid != neededuid:
+        os.seteuid(neededuid)
+    if os.geteuid() != neededuid:
+        raise Exception('Need to run as root or owner of /etc/confluent')
+    return curruid
+
+
+def initialize_ca():
+    ouid = normalize_uid()
+    try:
+        os.makedirs('/etc/confluent/ssh', mode=0o700)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+    finally:
+        os.seteuid(ouid)
+    myname = collective.get_myname()
+    caname = '{0} SSH CA'.format(myname)
+    subprocess.check_call(
+        ['ssh-keygen', '-C', caname, '-t', 'ed25519', '-f',
+         '/etc/confluent/ssh/ca', '-N', ''], preexec_fn=normalize_uid)
+    try:
+        os.makedirs('/var/lib/confluent/public/site/ssh/', mode=0o755)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+    cafilename = '/var/lib/confluent/public/site/ssh/{0}.ca'.format(myname)
+    shutil.copy('/etc/confluent/ssh/ca.pub', cafilename)
+    #    newent = '@cert-authority * ' + capub.read()
+
+
+def sign_host_key(pubkey, nodename):
+    tmpdir = tempfile.mkdtemp()
+    try:
+        pkeyname = os.path.join(tmpdir, 'hostkey.pub')
+        with open(pkeyname, 'wb') as pubfile:
+            pubfile.write(pubkey)
+        subprocess.check_call(
+            ['ssh-keygen', '-s', '/etc/confluent/ssh/ca', '-I', nodename,
+             '-n', nodename, '-h', pkeyname])
+        certname = pkeyname.replace('.pub', '-cert.pub')
+        with open(certname) as cert:
+            return cert.read()
+    finally:
+        shutil.rmtree(tmpdir)
+
+def initialize_root_key(generate):
+    authorized = []
+    myname = collective.get_myname()
+    for currkey in glob.glob('/root/.ssh/*.pub'):
+        authorized.append(currkey)
+    if generate and not authorized:
+        subprocess.check_call(['ssh-keygen', '-t', 'ed25519', '-f', '/root/.ssh/id_ed25519', '-N', ''])
+        for currkey in glob.glob('/root/.ssh/*.pub'):
+            authorized.append(currkey)
+    try:
+        os.makedirs('/var/lib/confluent/public/site/ssh', mode=0o755)
+        neededuid = os.stat('/etc/confluent').st_uid
+        os.chown('/var/lib/confluent', neededuid, -1)
+        os.chown('/var/lib/confluent/public', neededuid, -1)
+        os.chown('/var/lib/confluent/public/site', neededuid, -1)
+        os.chown('/var/lib/confluent/public/site/ssh', neededuid, -1)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+    neededuid = os.stat('/etc/confluent').st_uid
+    for auth in authorized:
+        shutil.copy(
+            auth,
+            '/var/lib/confluent/public/site/ssh/{0}.rootpubkey'.format(
+                    myname))
+        os.chmod('/var/lib/confluent/public/site/ssh/{0}.rootpubkey'.format(
+                myname), 0o644)
+        os.chown('/var/lib/confluent/public/site/ssh/{0}.rootpubkey'.format(
+                myname), neededuid, -1)
+
+
+def ca_exists():
+    return os.path.exists('/etc/confluent/ssh/ca')
+
+
+if __name__ == '__main__':
+    initialize_root_key(True)
+    if not ca_exists():
+        initialize_ca()
+    print(repr(sign_host_key(open('/etc/ssh/ssh_host_ed25519_key.pub').read(), collective.get_myname())))

confluent_server/confluent_server.spec.tmpl

@@ -13,7 +13,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}
 BuildArch: noarch
 %if "%{dist}" == ".el8"
-Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-pyte, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack
+Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-pyte, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack python3-libarchive-c
 %else
 Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client, python-pyparsing, python-paramiko, python-dns, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-pyte, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python2-websocket-client python2-msgpack
 %endif
@@ -58,7 +58,7 @@ true
 getent group confluent > /dev/null || /usr/sbin/groupadd -r confluent
 getent passwd confluent > /dev/null || /usr/sbin/useradd -r -g confluent -d /var/lib/confluent -s /sbin/nologin confluent
 mkdir -p /etc/confluent /var/lib/confluent /var/log/confluent /var/cache/confluent
-chown -R confluent:confluent /etc/confluent /var/lib/confluent /var/log/confluent /var/cache/confluent
+chown confluent:confluent /etc/confluent /var/log/confluent /var/cache/confluent
 
 %post
 sysctl -p /usr/lib/sysctl.d/confluent.conf >& /dev/null
@@ -73,7 +73,7 @@ if [ $NEEDCHOWN = 1 ]; then
         NEEDSTART=1
         systemctl stop confluent
     fi
-    chown -R confluent:confluent /etc/confluent /var/lib/confluent /var/log/confluent /var/cache/confluent
+    chown -R confluent:confluent /etc/confluent /var/log/confluent /var/cache/confluent
 fi
 systemctl daemon-reload
 if systemctl is-active confluent > /dev/null || [ $NEEDSTART = 1 ]; then /usr/bin/systemctl restart confluent >& /dev/null; fi

confluent_server/setup.py.tmpl

@@ -21,7 +21,7 @@ setup(
     install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet',
                       'dnspython', 'netifaces', 'pyte', 'pysnmp', 'pyparsing',
                       'pyghmi>=1.0.44'],
-    scripts=['bin/confluent', 'bin/confluentdbutil', 'bin/collective'],
+    scripts=['bin/confluent', 'bin/confluentdbutil', 'bin/collective', 'bin/osimage'],
     data_files=[('/etc/init.d', ['sysvinit/confluent']),
                 ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']),
                 ('/usr/lib/systemd/system', ['systemd/confluent.service']),

confluent_server/systemd/confluent.service

@@ -14,7 +14,7 @@ ConfigurationDirectory=confluent
 ExecStart=/opt/confluent/bin/confluent
 ExecStop=/opt/confluent/bin/confetty shutdown /
 Restart=on-failure
-AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_CHOWN
+AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_CHOWN CAP_NET_RAW
 User=confluent
 Group=confluent
 DevicePolicy=closed

misc/clortho.c

@@ -1,85 +0,0 @@
-/* Copyright 2019 Lenovo */
-#include <arpa/inet.h>
-#include <crypt.h>
-#include <net/if.h>
-#include <sys/socket.h>
-#include <stdio.h>
-#include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-
-#define OUI_ETHERTYPE 0x88b7
-#define MAXPACKET 1024
-#define CHDR "\xa4\x8c\xdb\x30\x01"
-
-int get_interface_index(int sock, char *interface) {
-        struct ifreq req;
-        memset(&req, 0, sizeof(req));
-        strncpy(req.ifr_name, interface, IFNAMSIZ);
-        if (ioctl(sock, SIOCGIFINDEX, &req) < 0) {
-                return -1;
-        }
-        return req.ifr_ifindex;
-}
-
-unsigned char* genpasswd() {
-        unsigned char * passwd;
-        int urandom;
-        passwd = calloc(33, sizeof(char));
-        urandom = open("/dev/urandom", O_RDONLY);
-        read(urandom, passwd, 32);
-        close(urandom);
-        for (urandom = 0; urandom < 32; urandom++) {
-                passwd[urandom] = 0x30 + (passwd[urandom] >> 2);
-        }
-        return passwd;
-
-}
-
-int parse_macaddr(char* macaddr) {
-        unsigned char *curr;
-        unsigned char idx;
-        curr = strtok(macaddr, ":-");
-        idx = 0;
-
-        while (curr != NULL) {
-                macaddr[idx++] = strtoul(curr, NULL, 16);
-                curr = strtok(NULL, ":-");
-        }
-
-}
-
-int main(int argc, char* argv[]) {
-        int sock;
-        int iface;
-        unsigned char* passwd;
-        unsigned char* macaddr;
-
-        unsigned char buffer[MAXPACKET];
-
-        passwd = genpasswd();
-        if (argc < 3) { 
-            fprintf(stderr, "Missing interface name and target MAC\n");
-            exit(1);
-        }
-        printf("%s\n", argv[2]);
-        parse_macaddr(argv[2]);
-        printf("%s\n", argv[2]);
-        sock = socket(AF_PACKET, SOCK_DGRAM, htons(OUI_ETHERTYPE));
-        if (sock < 0) {
-                fprintf(stderr, "Unable to open socket (run as root?)\n");
-                exit(1);
-        }
-        iface = get_interface_index(sock, argv[1]);
-        if (iface < 0) {
-                fprintf(stderr, "Unable to find specified interface '%s'\n", argv[1]);
-                exit(1);
-        }
-
-
-}
-

misc/getcsr.py

@@ -0,0 +1,46 @@
+import argparse
+import io
+import gzip
+import pyghmi.redfish.command as cmd
+import os
+import sys
+
+ap = argparse.ArgumentParser(description='Certificate Generate')
+ap.add_argument('xcc', help='XCC address')
+ap.add_argument('--country', help='Two Letter Country')
+ap.add_argument('--state', help='State or Province')
+ap.add_argument('--city', help='City or Locality')
+ap.add_argument('--org', help='Organization name')
+ap.add_argument('--name', help='Common/Host Name')
+args = ap.parse_args()
+
+c = cmd.Command(args.xcc, os.environ['XCCUSER'], os.environ['XCCPASS'],
+                verifycallback=lambda x: True)
+params = [
+        '0', # 'serviceType'
+        args.country,
+        args.state,
+        args.city,
+        args.org,
+        args.name,
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+]
+wc = c.oem.wc
+rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_GenKeyAndCSR': ','.join(params)})
+rsp, status = wc.grab_json_response_with_status('/api/dataset', {'CSR_Format': '1'})
+rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_DownloadCSRANDCert': '0,4,0'})
+wc.request('GET', '/download/{0}'.format(rsp['FileName']))
+rsp = wc.getresponse()
+csr = rsp.read()
+if rsp.getheader('Content-Encoding', None) == 'gzip':
+    csr = gzip.GzipFile(fileobj=io.BytesIO(csr)).read()
+print(csr)
+        

misc/getinstdisk.py

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

misc/installcert.py

@@ -0,0 +1,16 @@
+import argparse
+import pyghmi.redfish.command as cmd
+import os
+import sys
+
+ap = argparse.ArgumentParser(description='Certificate Generate')
+ap.add_argument('xcc', help='XCC address')
+ap.add_argument('cert', help='Certificate in PEM format')
+args = ap.parse_args()
+
+c = cmd.Command(args.xcc, os.environ['XCCUSER'], os.environ['XCCPASS'],
+                verifycallback=lambda x: True)
+wc = c.oem.wc
+cert = open(args.cert, 'rb').read()
+res = wc.grab_json_response_with_status('/api/function', {'Sec_ImportCert': '0,1,0,0,,{0}'.format(cert)
+print(repr(res))

misc/makeboot.centos.py

@@ -0,0 +1,60 @@
+#!/usr/bin/python
+
+import os
+import sys
+import yaml
+
+def mkdirp(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+
+def makeboot_tree(distribution, profiledir):
+    bootdir = os.path.join(profiledir, 'boot')
+    efisrc = os.path.join(distribution, 'EFI')
+    efidir = os.path.join(bootdir, 'efi/boot')
+    mkdirp(efidir)
+    initrfsdir = os.path.join(bootdir, 'initramfs')
+    mkdirp(initrfsdir)
+    for directory in os.walk(efisrc):
+        for filename in directory[2]:
+            if filename.lower() == 'bootx64.efi':
+                srcfile = os.path.join(directory[0], filename)
+                trgfile = os.path.join(efidir, 'bootx64.efi')
+                os.link(srcfile, trgfile)
+            elif filename.lower() == 'grubx64.efi':
+                srcfile = os.path.join(directory[0], filename)
+                trgfile = os.path.join(efidir, 'grubx64.efi')
+                os.link(srcfile, trgfile)
+    netbootdir = os.path.join(distribution, 'images/pxeboot')
+    srckern = os.path.join(netbootdir, 'vmlinuz')
+    srcinitramfs = os.path.join(netbootdir, 'initrd.img')
+    trgkern = os.path.join(bootdir, 'kernel')
+    trginitramfs = os.path.join(initrfsdir, 'initrd.img')
+    os.link(srckern, trgkern)
+    os.link(srcinitramfs, trginitramfs)
+    trginitramfs = os.path.join(initrfsdir, 'site-initramfs.gz')
+    os.link('/var/lib/confluent/public/site/site-initramfs.gz', trginitramfs)
+    profileinfo = os.path.join(profiledir, 'profile.yaml')
+    with open(profileinfo) as info:
+        profile = yaml.load(info)
+    cfgfile = os.path.join(efidir, 'grub.cfg')
+    with open(cfgfile, 'w') as grubcfg:
+        grubcfg.write('set timeout=5\n')
+        grubcfg.write("menuentry '{0}' {{\n".format(
+            profile.get('label', 'Unknown')))
+        grubcfg.write('    linuxefi /kernel {0}\n'.format(profile.get(
+                     'kernelargs', 'quiet')))
+        initrds = ' '.join(
+            ['/initramfs/{0}'.format(x) for x in os.listdir(initrfsdir)])
+        grubcfg.write('    initrdefi {0}\n}}'.format(initrds))
+    #TODO: create the netboot grub.cfg, is there a way to use grub http
+    # without putting the server in the cfg?
+    # If server is omitted, value of environment variable ‘net_default_server’
+    # is used
+
+
+if __name__ == '__main__':
+    makeboot_tree(sys.argv[1], sys.argv[2])