Fix indentation mistake in add_local_repositories

If multiple things try to renam to the same thing, block the action.

confluent_client/COPYRIGHT

@@ -0,0 +1,21 @@
+Name: confluent-client
+Project: https://hpc.lenovo.com/users/
+Source: https://github.com/lenovo/confluent
+
+Files: *
+Copyright: 2014-2019 Lenovo
+License: Apache-2.0
+
+File: client.py:
+Copyright: 2014 IBM Corporation
+           2015-2019 Lenovo
+License: Apache-2.0
+
+File: sortutil.py
+Copyright: 2014 IBM Corporation
+           2015-2016 Lenovo
+License: Apache-2.0
+
+File: tlv.py
+Copyright: 2014 IBM Corporation
+License: Apache-2.0

confluent_client/bin/confetty

@@ -635,7 +635,7 @@ def quitconfetty(code=0, fullexit=False, fixterm=True):
     global didconsole
     if fixterm or didconsole:
         currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
-        fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
+        fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl & ~os.O_NONBLOCK)
         if oldtcattr is not None:
             termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
     # Request default color scheme, to undo potential weirdness of terminal

confluent_client/bin/confluent2xcat

@@ -79,6 +79,8 @@ def collect_attribute_data(databynode, node):
     xcatattrs['groups'] = ','.join(data.get('groups', []))
     xcatattrs['bmc'] = data.get('hardwaremanagement.manager', {}).get(
         'value', None)
+    if xcatattrs['bmc']:
+        xcatattrs['bmc'] = xcatattrs['bmc'].split('/', 1)[0]
     xcatattrs['mpa'] = data.get('enclosure.manager', {}).get(
         'value', None)
     xcatattrs['slotid'] = data.get('enclosure.bay', {}).get(

confluent_client/bin/nodeconsole

@@ -38,6 +38,26 @@ argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
 argparser.add_option('-l', '--log', action='store_true', default=False,
                      help='Enter log replay mode instead of showing a live console')
+argparser.add_option('-w','--windowed', action='store_true', default=False,
+                     help='Open terminal windows for each node.  The '
+                          'environment variable NODECONSOLE_WINDOWED_COMMAND '
+                          'should be set, which should be a text string corresponding '
+                          'to a command that can be used to open a windowed console,'
+                          ' omitting the "nodeconsole <noderange>" part of the '
+                          'command, for example, to open a set of consoles for a '
+                          'range of nodes in separate xterm windows, set '
+                          'NODECONSOLE_WINDOWED_COMMAND to "xterm -e".  To open a '
+                          'set of consoles for a range of nodes in separate '
+                          'GNOME Terminal windows with a size of 100 columns and '
+                          '31 rows, set NODECONSOLE_WINDOWED_COMMAND '
+                          'to "gnome-terminal --geometry 100x31 --" or in a WSL '
+                          'environment, to open a set of consoles for a range of '
+                          'nodes in separate Windows Terminal windows, with the '
+                          'title set for each node, set NODECONSOLE_WINDOWED_COMMAND'
+                          ' to "wt.exe wsl.exe -d AlmaLinux-8 '
+                          '--shell-type login".  If the NODECONSOLE_WINDOWED_COMMAND '
+                          'environment variable isn\'t set, xterm will be used by'
+                          'default.')
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
@@ -51,6 +71,27 @@ if options.log:
         sys.exit(1)
     logreader.replay_to_console(logname)
     sys.exit(0)
+#added functionality for wcons
+
+if options.windowed:
+    envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
+    if not envstring:
+        envlist=["xterm", "-e"]
+    else:
+        envlist=os.environ.get('NODECONSOLE_WINDOWED_COMMAND').split(' ')
+    nodes = []
+    sess = client.Command()
+    for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
+        node = res.get('item', {}).get('href', '/').replace('/', '')
+        if not node:
+            sys.stderr.write(res.get('error', repr(res)) + '\n')
+            sys.exit(1)
+        nodes.append(node)
+    for node in sortutil.natural_sort(nodes):
+        with open(os.devnull, 'wb') as devnull:
+            subprocess.Popen(envlist + [confettypath, '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)], stdin=devnull)
+    sys.exit(0)
+#end of wcons
 if options.tile:
     null = open('/dev/null', 'w')
     nodes = []

confluent_client/bin/nodedeploy

@@ -86,6 +86,10 @@ def main(args):
     c = client.Command()
     c.stop_if_noderange_over(args.noderange, args.maxnodes)
     errnodes = set([])
+    for rsp in c.read('/noderange/{0}/nodes/'.format(args.noderange)):
+        if 'error' in rsp:
+            sys.stderr.write(rsp['error'] + '\n')
+            sys.exit(1)
     if not args.clear and args.network and not args.prepareonly:
         rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
                                         bootmode='uefi',

confluent_client/bin/nodediscover

@@ -50,19 +50,52 @@ columnmapping = {
 }
 #TODO: add chassis uuid
 
+def register_endpoint(options, session, addr):
+    neednewline = False
+    current = 0
+    for rsp in session.update('/discovery/register', {'addresses': addr}):
+        if 'count' in rsp:
+            total = rsp['count']
+        elif total > 1:
+            current += 1
+            sys.stdout.write('\x1b[2K\r{0:.1f}% Scanned'.format(100 * current / total))
+            sys.stdout.flush()
+            neednewline = True
+        if 'created' in rsp:
+            if neednewline:
+                print('')
+                neednewline = False
+            print('Registered: {0}'.format(rsp['created']))
+    if neednewline:
+        print('')
 
+def subscribe_discovery(options, session, subscribe, targ):
+    keyn = 'subscribe' if subscribe else 'unsubscribe'
+    payload = {keyn: targ}
+    if subscribe:
+        for rsp in session.update('/discovery/subscriptions/{0}'.format(targ), payload):
+            if 'status' in rsp:
+                print(rsp['status'])
+    else:
+        for rsp in session.delete('/discovery/subscriptions/{0}'.format(targ)):
+            if 'status' in rsp:
+                print(rsp['status'])
+    
 def print_disco(options, session, currmac, outhandler, columns):
     procinfo = {}
     for tmpinfo in session.read('/discovery/by-mac/{0}'.format(currmac)):
 
         procinfo.update(tmpinfo)
     if 'Switch' in columns or 'Port' in columns:
-        for tmpinfo in session.read(
-                '/networking/macs/by-mac/{0}'.format(currmac)):
-            if 'ports' in tmpinfo:
-                # The api sorts so that the most specific available value
-                # is last
-                procinfo.update(tmpinfo['ports'][-1])
+        if 'switch' in procinfo:
+            procinfo['port'] = procinfo['switchport']
+        else:
+            for tmpinfo in session.read(
+                    '/networking/macs/by-mac/{0}'.format(currmac)):
+                if 'ports' in tmpinfo:
+                    # The api sorts so that the most specific available value
+                    # is last
+                    procinfo.update(tmpinfo['ports'][-1])
     record = []
     for col in columns:
         rawcol = columnmapping[col]
@@ -320,7 +353,7 @@ def blocking_scan(session):
 
 def main():
     parser = optparse.OptionParser(
-        usage='Usage: %prog [list|assign|rescan|clear] [options]')
+        usage='Usage: %prog [list|assign|rescan|clear|subscribe|unsubscribe|register] [options]')
     # -a for 'address' maybe?
     # order by
     # show state (discovered or..
@@ -358,9 +391,15 @@ def main():
     parser.add_option('-o', '--order', dest='order',
                       help='Order output by given field', metavar='ORDER')
     (options, args) = parser.parse_args()
-    if len(args) == 0 or args[0] not in ('list', 'assign', 'reassign', 'rescan', 'clear'):
+    if len(args) == 0 or args[0] not in ('list', 'assign', 'reassign', 'rescan', 'clear', 'subscribe', 'unsubscribe', 'register'):
         parser.print_help()
         sys.exit(1)
+    if args[0] == 'register' and len(args) != 2:
+        sys.stderr.write('Register requires target address or range (e.g. 192.168.2.30, 192.168.2.0/24, or 192.168.2.1-192.168.2.30)\n')
+        sys.exit(1)
+    if 'subscribe' in args[0] and len(args) != 2:
+        sys.stderr.write('subscribe/unsubscribe subcommands require switch name as argument\n')
+        sys.exit(1)
     session = client.Command()
     if args[0] == 'list':
         list_discovery(options, session)
@@ -370,6 +409,12 @@ def main():
         assign_discovery(options, session)
     if args[0] == 'reassign':
         assign_discovery(options, session, False)
+    if args[0] == 'register':
+        register_endpoint(options, session, args[1])
+    if args[0] == 'subscribe':
+        subscribe_discovery(options, session, True, args[1])
+    if args[0] == 'unsubscribe':
+        subscribe_discovery(options, session, False, args[1])
     if args[0] == 'rescan':
         blocking_scan(session)
         print("Rescan complete")

confluent_client/bin/nodegroupattrib

@@ -60,10 +60,12 @@ nodetype="nodegroups"
 
 try:
     nodegroups = args[0]
+    client.check_globbing(nodegroups)
     nodelist = '/{0}/{1}/'.format(nodetype,nodegroups)
 except IndexError:
+    argparser.print_help()
+    sys.exit(1)
     nodelist = '/nodegroups/'
-client.check_globbing(nodegroups)
 session = client.Command()
 exitcode = 0
 

confluent_client/bin/nodersync

@@ -46,6 +46,8 @@ def run():
                      help='Specify a maximum number of '
                           'nodes to run rsync to, '
                           'prompting if over the threshold')   
+    argparser.add_option('-l', '--loginname', type='str',
+                         help='Username to use when connecting, defaults to current user.')
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -61,7 +63,10 @@ def run():
     c = client.Command()
     cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
-    cmdstr += ' {node}:' + targpath
+    if options.loginname:
+        cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath
+    else:
+        cmdstr += ' {node}:' + targpath
 
     currprocs = 0
     all = set([])

confluent_client/confluent/logreader.py

@@ -282,14 +282,19 @@ def _replay_to_console(txtfile, binfile):
 def replay_to_console(txtfile):
     if os.path.exists(txtfile + '.cbl'):
         binfile = txtfile + '.cbl'
+    elif '.' not in txtfile:
+        if '/' not in txtfile:
+            txtfile = os.getcwd() + '/' + txtfile
+        sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(txtfile + '.cbl'))
+        sys.exit(1)
     else:
         fileparts = txtfile.split('.')
         prefix = '.'.join(fileparts[:-1])
         binfile = prefix + '.cbl.' + fileparts[-1]
         if not os.path.exists(binfile):
-            sys.stderr.write('Unable to locate cbl file\n')
+            sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(binfile))
             sys.exit(1)
     _replay_to_console(txtfile, binfile)
 
 if __name__ == '__main__':
-    replay_to_console(sys.argv[1])
+    replay_to_console(sys.argv[1])

confluent_client/confluent/screensqueeze.py

@@ -29,6 +29,9 @@ class ScreenPrinter(object):
         self.client = client
         self.nodeoutput = {}
         self.nodelist = []
+        self.nodepos = {}
+        self.lastrows = 0
+        self.fieldchanged = False
         maxlen = 0
         for ans in self.client.read('/noderange/{0}/nodes/'.format(noderange)):
             if 'error' in ans:
@@ -50,16 +53,19 @@ class ScreenPrinter(object):
         if len(text) >= self.textlen:
             self.textlen = len(text) + 1
             self.fieldwidth = self.textlen + self.nodenamelen + 1
-        self.drawscreen()
+            self.fieldchanged = True
+        self.drawscreen(node)
 
-    def drawscreen(self):
+    def drawscreen(self, node=None):
         if self.squeeze:
             currheight, currwidth = get_screengeom()
-            currheight -= 1
+            currheight -= 2
+            if currheight < 1:
+                currheight = 1
             numfields = currwidth // self.fieldwidth
             fieldformat = '{{0:>{0}}}:{{1:{1}}}'.format(self.nodenamelen,
                                                         self.textlen)
-            sys.stdout.write('\x1b[2J\x1b[;H')  # clear screen
+            #sys.stdout.write('\x1b[2J\x1b[;H')  # clear screen
             if len(self.nodelist) < (numfields * currheight):
                 numfields = len(self.nodelist) // currheight + 1
         else:
@@ -67,16 +73,40 @@ class ScreenPrinter(object):
             fieldformat = '{0}: {1}'
         if self.squeeze:
             columns = [self.nodelist[x:x+currheight] for x in range(0, len(self.nodelist), currheight)]
-            for currow in range(0, len(columns[0])):
-                for col in columns:
-                    try:
-                        node = col[currow]
-                        sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
-                    except IndexError:
-                        break
-                sys.stdout.write('\n')
+            if self.lastrows:
+                sys.stdout.write('\x1b[{0}A'.format(self.lastrows))
+            if node and self.lastrows == len(columns[0]) and not self.fieldchanged:
+                targline, targcol = self.nodepos[node]
+                if targline:
+                    sys.stdout.write('\x1b[{0}B'.format(targline))
+                if targcol:
+                    sys.stdout.write('\x1b[{0}C'.format(targcol))
+                sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
+                sys.stdout.write('\r\x1b[{0}B'.format(self.lastrows - targline))
+            else:
+                self.lastrows = 0
+                self.fieldchanged = False
+                column = 0
+                for currow in range(0, len(columns[0])):
+                    sys.stdout.write('\x1b[2K')
+                    for col in columns:
+                        try:
+                            node = col[currow]
+                            self.nodepos[node] = (currow, column)
+                            sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
+                            column += len(fieldformat.format(node, self.nodeoutput[node]))
+                        except IndexError:
+                            break
+                    sys.stdout.write('\n')
+                    column = 0
+                    self.lastrows += 1
+                sys.stdout.write('\x1b[J')
         else:
-            for node in self.nodelist:
+            if node:
+                nodes = [node]
+            else:
+                nodes = self.nodelist
+            for node in nodes:
                 sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
                 sys.stdout.write('\n')
         sys.stdout.flush()

confluent_client/confluent_client.spec.tmpl

@@ -12,8 +12,8 @@ Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}
 BuildArch: noarch
-Vendor: Jarrod Johnson <jjohnson2@lenovo.com>
-Url: http://xcat.sf.net/
+Vendor: Lenovo
+Url: http://github.com/lenovo/confluent
 Obsoletes: confluent_common
 
 %description
@@ -24,25 +24,17 @@ a confluent server.
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-%if "%{dist}" == ".el8"
-python3 setup.py build
-%else
-%if "%{dist}" == ".el9"
-python3 setup.py build
-%else
+%if "%{dist}" == ".el7"
 python2 setup.py build
-%endif
+%else
+python3 setup.py build
 %endif
 
 %install
-%if "%{dist}" == ".el8"
-python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
-%else
-%if "%{dist}" == ".el9"
-python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
-%else
+%if "%{dist}" == ".el7"
 python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
-%endif
+%else
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
 %endif
 
 
@@ -51,4 +43,6 @@ rm -rf $RPM_BUILD_ROOT
 
 %files -f INSTALLED_FILES
 %license /opt/confluent/share/licenses/confluent_client/LICENSE
+%license /opt/confluent/share/licenses/confluent_client/COPYRIGHT
 %defattr(-,root,root)
+/opt/confluent

confluent_client/confluent_env.sh

@@ -82,6 +82,17 @@ function _confluent_generic_completion()
         return;
     fi
 }
+function _confluent_generic_ng_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_ng_completion
+        return;
+    fi
+}
 _confluent_nodeidentify_completion()
 {
     COMP_CANDIDATES=("on,off,blink -h")
@@ -142,7 +153,7 @@ _confluent_osimage_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 2 ]; then
-        COMPREPLY=($(compgen -W "initialize import updateboot" -- ${COMP_WORDS[COMP_CWORD]}))
+        COMPREPLY=($(compgen -W "initialize import updateboot rebase" -- ${COMP_WORDS[COMP_CWORD]}))
         return
     elif [ ${CMPARGS[1]} == 'initialize' ]; then
         COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]}))
@@ -150,7 +161,7 @@ _confluent_osimage_completion()
         compopt -o default
         COMPREPLY=()
         return
-    elif [ ${CMPARGS[1]} == 'updateboot' ]; then
+    elif [ ${CMPARGS[1]} == 'updateboot' -o ${CMPARGS[1]} == 'rebase' ]; then
         COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}"))
         return
     fi
@@ -268,7 +279,20 @@ _confluent_nodeattrib_completion()
     _confluent_generic_completion
 }
 
+_confluent_define_completion()
+{
+    COMP_CANDIDATES=$(nodegroupattrib 'everything' all | awk '{print $2}'|sed -e 's/://')
+    _confluent_get_args
+    if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
+    fi
+}
 
+_confluent_nodegroupattrib_completion()
+{
+    COMP_CANDIDATES=$(nodegroupattrib 'everything' all | awk '{print $2}'|sed -e 's/://')
+    _confluent_generic_ng_completion
+}
 _confluent_nn_completion()
 {
     _confluent_get_args
@@ -323,18 +347,30 @@ _confluent_ng_completion()
 
     COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
+
+_confluent_nodediscover_completion()
+{
+        _confluent_get_args
+        if [ $NUMARGS == 2 ]; then
+            COMPREPLY=($(compgen -W "list assign rescan clear subscribe unsubscribe register" -- ${COMP_WORDS[COMP_CWORD]}))
+            return;
+        fi
+
+}
+complete -F _confluent_nodediscover_completion nodediscover
 complete -F _confluent_nodeattrib_completion nodeattrib
-complete -F _confluent_nodeattrib_completion nodegroupattrib
 complete -F _confluent_nr_completion nodebmcreset
 complete -F _confluent_nodesetboot_completion nodeboot
 complete -F _confluent_nr_completion nodeconfig
 complete -F _confluent_nn_completion nodeconsole
+complete -F _confluent_define_completion nodedefine
+complete -F _confluent_define_completion nodegroupdefine
 complete -F _confluent_nr_completion nodeeventlog
 complete -F _confluent_nodefirmware_completion nodefirmware
 complete -F _confluent_nodedeploy_completion nodedeploy
 complete -F _confluent_osimage_completion osdeploy
 complete -F _confluent_imgutil_completion imgutil
-complete -F _confluent_ng_completion nodegroupattrib
+complete -F _confluent_nodegroupattrib_completion nodegroupattrib
 complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth
 complete -F _confluent_nodeidentify_completion nodeidentify

confluent_client/doc/man/confluent2hosts.ronn

@@ -0,0 +1,28 @@
+confluent2hosts(8) -- Generate /etc/hosts entries for nodes
+========================================================
+
+## SYNOPSIS
+
+`confluent2hosts -i <ip expression> -n <name expression> <noderange>`  
+`confluent2hosts -a <noderange>`  
+
+## DESCRIPTION
+
+`confluent2hosts` can be used to help generate /etc/hosts entries for a 
+noderange.  There are two general approaches.
+
+It can be used ad-hoc, using -i and -n to specify the address and name portions respectively.  This accepts the standard confluent expression syntax, allowing for things like 172.30.1.{n1} or {node}.{dns.domain} or {bmc}.
+
+It can also read from the confluent db, using `-a`.  In this mode, each net.<value>.<attribute> group is pulled together into hosts lines.  ipv4_address and ipv6_address fields are associated with the corresponding hostname attributes.
+
+## EXAMPLES
+
+* Generate /etc/hosts entries ad-hoc using default name:
+  `# confluent2hosts -i 10.2.3.{n1} d9-d12`  
+
+* Generate /etc/hosts entries ad-hoc using alternative name:
+  `# confluent2hosts -i 10.2.3.{n1} -n "{node}-alt {node}-alt.{dns.domain}" d9-d12`  
+
+* Generate /etc/hosts entries using the confluent DB as a reference:
+  `# confluent2hosts -a d9-d12`  
+

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -83,12 +83,12 @@ to a blank value will allow masking a group defined attribute with an empty valu
     `# nodeattrib n1,n2 hardwaremanagement --blame`  
     `n1: hardwaremanagement.manager: 172.30.3.1`  
     `n1: hardwaremanagement.method: ipmi (inherited from group everything)`  
-    `n1: hardwaremanagement.switch: r8e1`  
-    `n1: hardwaremanagement.switchport: 14`  
+    `n1: net.switch: r8e1`  
+    `n1: net.switchport: 14`  
     `n2: hardwaremanagement.manager: 172.30.3.2`  
     `n2: hardwaremanagement.method: ipmi (inherited from group everything)`  
-    `n2: hardwaremanagement.switch: r8e1`  
-    `n2: hardwaremanagement.switchport: 2`  
+    `n2: net.switch: r8e1`  
+    `n2: net.switchport: 2`  
 
 * Listing matching nodes of a simple noderange that are set:
     `# nodeattrib n1-n2 current`  

confluent_client/doc/man/nodeconsole.ronn

@@ -26,6 +26,26 @@ a new session.
   Perform a log reply on the current, local log in /var/log/confluent/consoles.
   If in collective mode, this only makes sense to use on the current collective
   manager at this time.
+  
+* `-w`, `--windowed`:
+  Open terminal windows for each node.  The
+  environment variable **NODECONSOLE_WINDOWED_COMMAND**
+  should be set, which should be a text string corresponding
+  to a command that can be used to open a windowed console,
+  omitting the `nodeconsole <noderange>` part of the
+  command, for example, to open a set of consoles for a
+  range of nodes in separate xterm windows, set
+  **NODECONSOLE_WINDOWED_COMMAND** to `xterm -e`.  To open a
+  set of consoles for a range of nodes in separate
+  GNOME Terminal windows with a size of 100 columns and
+  31 rows, set **NODECONSOLE_WINDOWED_COMMAND**
+  to `gnome-terminal --geometry 100x31 --` or in a WSL
+  environment, to open a set of consoles for a range of
+  nodes in separate Windows Terminal windows, with the
+  title set for each node, set **NODECONSOLE_WINDOWED_COMMAND**
+  to `wt.exe wsl.exe -d AlmaLinux-8 --shell-type login.  If the
+  NODECONSOLE_WINDOWED_COMMAND environment variable isn't set,
+  xterm will be used bydefault.
 
 ## ESCAPE SEQUENCE COMMANDS
 

confluent_client/doc/man/nodediscover.ronn

@@ -5,9 +5,12 @@ nodediscover(8)  -- List or manage confluent node discovery
 
 `nodediscover rescan`  
 `nodediscover [options] list`  
-`nodediscover [options] assign`
-`nodediscover [options] rescan`
-`nodediscover [options] clear`
+`nodediscover [options] assign`  
+`nodediscover [options] rescan`  
+`nodediscover [options] clear`  
+`nodediscover [options] subscribe [switch]`  
+`nodediscover [options] unsubscribe [switch]`  
+`nodediscover [options] register [Addresses, e.g. 192.168.1.2 or 192.168.1.0/24 or 192.168.1.1-192.168.1.28]`  
 
 ## DESCRIPTION
 
@@ -35,6 +38,12 @@ scan is the best approach to catch something that appears to be missing.
 **nodedsicover clear** requests the server forget about a selection of
 detected device.  It takes the same arguments as **nodediscover list**.
 
+**nodediscover subscribe** and **unsubscribe** instructs confluent to subscribe to or
+unsubscribe from the designated switch running affluent with system discovery support.
+
+**nodediscover register** instructs confluent to perform a remote probe of an address, subnet,
+or range of IP addresses.
+
 ## CSV FORMAT
 
 The CSV format used by nodediscover consists of one header describing the

confluent_client/doc/man/osdeploy.ronn

@@ -5,6 +5,7 @@ osdeploy(8) --- Configure general OS deployment facilities of confluent
 
 `osdeploy import <iso>`  
 `osdeploy updateboot <profile>`  
+`osdeploy rebase <profile>`  
 `osdeploy initialize [-h] [-g] [-u] [-s] [-k] [-t] [-p] [-i] [-l] [-a]`  
 
 ## DESCRIPTION
@@ -13,5 +14,6 @@ osdeploy(8) --- Configure general OS deployment facilities of confluent
 `import` subcommand will generate profiles from an iso image. `updateboot` will take any
 changes in the specified update that need to be pushed into boot configuration and/or images
 and ensure those needed changes are performed. `initialize` provides assistance in setting
-up the most commonly required facilities. Run `osdeploy initialize -h` for more detail
-on the options offered by `osdeploy initialize`
+up the most commonly required facilities. `rebase` has confluent attempt to update profile
+content that came from /opt/confluent in an rpm update, if profile supports it. 
+Run `osdeploy initialize -h` for more detail on the options offered by `osdeploy initialize`.

confluent_client/setup.py.tmpl

@@ -2,7 +2,7 @@ from setuptools import setup
 import os
 
 data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh']),
-              ('/opt/confluent/share/licenses/confluent_client/', ['LICENSE'])
+              ('/opt/confluent/share/licenses/confluent_client/', ['LICENSE', 'COPYRIGHT'])
               ]
 try:
     scriptlist = ['bin/{0}'.format(d) for d in os.listdir('bin/')]
@@ -15,10 +15,14 @@ except OSError:
 setup(
     name='confluent_client',
     version='#VERSION#',
-    author='Jarrod Johnson',
+    author='Lenovo',
     author_email='jjohnson2@lenovo.com',
-    url='http://xcat.sf.net/',
+    url='http://github.com/lenovo/confluent/',
+    license='Apache-2.0',
+    description='Command line client and libraries for confluent management server',
+    long_description='Command line client and libraries for confluent management server',
     packages=['confluent'],
+    platforms=['Linux'],
     scripts=scriptlist,
     data_files=data_files,
 )

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -159,6 +159,7 @@ def get_net_apikey(nodename, mgr):
             else:
                 cliaddr = ('::', 302)
             clisock.bind(cliaddr)
+            clisock.settimeout(1)
             clisock.connect(addrinfo[-1])
             rsp = clisock.recv(8)
             if rsp != b'\xc2\xd1-\xa8\x80\xd8j\xba':
@@ -222,7 +223,8 @@ def get_apikey(nodename, hosts, errout=None):
     return apikey
 
 class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, usejson=False, port=443, host=None, errout=None):
+    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None):
+        self.phmac = phmac
         self.errout = None
         if errout:
             self.errout = open(errout, 'w')
@@ -243,24 +245,34 @@ class HTTPSClient(client.HTTPConnection, object):
             self.hosts = []
             info = open('/etc/confluent/confluent.info').read().split('\n')
             havedefault = '0'
+            plainhost = ''
             for line in info:
+                host = ''
                 if line.startswith('NODENAME:'):
                     node = line.split(' ')[1]
                     self.stdheaders['CONFLUENT_NODENAME'] = node
                 if line.startswith('MANAGER:') and not host:
                     host = line.split(' ')[1]
+                    if not plainhost:
+                        plainhost = host
                 if line.startswith('EXTMGRINFO:'):
                     extinfo = line.split(' ')[1]
                     extinfo = extinfo.split('|')
-                    if not mgtiface:
-                        host, mgtiface, havedefault = extinfo[:3]
                     if havedefault == '0' and extinfo[2] == '1':
                         host, mgtiface, havedefault = extinfo[:3]
-            if '%' in host:
-                ifidx = host.split('%', 1)[1]
-                with open('/tmp/confluent.ifidx', 'w+') as ifout:
-                    ifout.write(ifidx)
-            self.hosts.append(host)
+                        if '%' in host:
+                            ifidx = host.split('%', 1)[1]
+                            with open('/tmp/confluent.ifidx', 'w+') as ifout:
+                                ifout.write(ifidx)    
+                    if not mgtiface:
+                        host, mgtiface, havedefault = extinfo[:3]
+                    if host:
+                        if havedefault == '0':
+                            if '%' in host:
+                                ifidx = host.split('%', 1)[1]
+                                with open('/tmp/confluent.ifidx', 'w+') as ifout:
+                                    ifout.write(ifidx)
+                        self.hosts.append(host)
             try:
                 info = open('/etc/confluent/confluent.deploycfg').read().split('\n')
             except Exception:
@@ -269,7 +281,13 @@ class HTTPSClient(client.HTTPConnection, object):
                 for line in info:
                     if line.startswith('deploy_server: ') or line.startswith('deploy_server_v6: '):
                         self.hosts.append(line.split(': ', 1)[1])
-        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
+            if plainhost and not self.hosts:
+                self.hosts.append(plainhost)
+        if self.phmac:
+            with open(phmac, 'r') as hmacin:
+                self.stdheaders['CONFLUENT_CRYPTHMAC'] = hmacin.read()
+        else:
+            self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
         self.port = port
@@ -406,6 +424,19 @@ if __name__ == '__main__':
         errout = sys.argv.pop(errout)
     except ValueError:
         errout = None
+    try:
+        phmac = sys.argv.index('-p')
+        sys.argv.pop(phmac)
+        phmac = sys.argv.pop(phmac)
+    except ValueError:
+        phmac = None
+    try:
+        checkonly = False
+        idxit = sys.argv.index('-c')
+        sys.argv.pop(idxit)
+        checkonly = True
+    except Exception:
+        pass
     if len(sys.argv) > 2 and os.path.exists(sys.argv[-1]):
         data = open(sys.argv[-1]).read()
     if outbin:
@@ -417,11 +448,13 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
+    client = HTTPSClient(usejson, errout=errout, phmac=phmac)
     if waitfor:
-        client = HTTPSClient(usejson, errout=errout)
         status = 201
         while status != waitfor:
             status, rsp = client.grab_url_with_status(sys.argv[1], data)
             sys.stdout.write(rsp.decode())
+    elif checkonly:
+        sys.stdout.write(client.check_connections())
     else:
-        sys.stdout.write(HTTPSClient(usejson, errout=errout).grab_url(sys.argv[1], data).decode())
+        sys.stdout.write(client.grab_url(sys.argv[1], data).decode())

confluent_osdeploy/common/profile/scripts/confignet

@@ -25,6 +25,8 @@ def add_lla(iface, mac):
     initbyte = int(pieces[0], 16) ^ 2
     lla = 'fe80::{0:x}{1}:{2}ff:fe{3}:{4}{5}/64'.format(initbyte, pieces[1], pieces[2], pieces[3], pieces[4], pieces[5])
     try:
+        with open('/proc/sys/net/ipv6/conf/{0}/disable_ipv6'.format(iface), 'w') as setin:
+            setin.write('0')
         subprocess.check_call(['ip', 'addr', 'add', 'dev', iface, lla, 'scope', 'link'])
     except Exception:
         return None

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -73,11 +73,13 @@ mv ../addons.tgz .
 cd ..
 cp -a esxi7out esxi6out
 cp -a esxi7 esxi6
+cp -a esxi7out esxi8out
+cp -a esxi7 esxi8
 
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 coreos; do
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh

@@ -20,6 +20,7 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     else
         v4addr=$(grep ^ipv4_address: $tcfg)
         v4addr=${v4addr#ipv4_address: }
+        v4addr=${v4addr%/*}
         v4gw=$(grep ^ipv4_gateway: $tcfg)
         v4gw=${v4gw#ipv4_gateway: }
         if [ "$v4gw" = "null" ]; then

confluent_osdeploy/debian/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -0,0 +1,97 @@
+reverse_uuid() {
+    echo $1 | head -c 8 | tail -c 2
+    echo $1 | head -c 6 | tail -c 2
+    echo $1 | head -c 4 | tail -c 2
+    echo $1 | head -c 2 | tail -c 2
+    echo $1 | head -c 13 | tail -c 2
+    echo $1 | head -c 11 | tail -c 2
+    echo $1 | head -c 18 | tail -c 2
+    echo $1 | head -c 16 | tail -c 2
+    echo $1 | tail -c 18 | sed -e s/-//
+}
+setdebopt() {
+    debconf-set $1 $2
+    echo d-i $1 $3 $2 >> /preseed.cfg
+}
+
+dhuuid=$(reverse_uuid $(cat /sys/devices/virtual/dmi/id/product_uuid))
+dhcpid=$(mktemp)
+mkdir -p /etc/confluent
+cp /tls/* /etc/ssl/certs/
+for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
+     ip link set $nic up
+done
+for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
+    if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
+         /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+         if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
+             break
+         fi
+    fi
+    ip -4 flush dev $nic
+done    
+mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+/opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+apikey=$(cat /etc/confluent/confluent.apikey)
+cd /etc/confluent
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg
+cd -
+predir=$(mktemp -d)
+cd $predir
+profile=$(grep ^profile: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh
+chmod u+x pre.sh
+wget https://$mgr/confluent-public/os/$profile/preseed.cfg
+mv preseed.cfg /
+setdebopt auto-install/enable true boolean
+setdebopt partman/early_command  $predir/pre.sh string
+cd -
+ip -4 a flush dev $nic
+setdebopt netcfg/choose_interface $nic select
+setdebopt netcfg/get_hostname $nodename string
+v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //')
+if [ "$v4cfgmeth" = "static" ]; then
+    setdebopt netcfg/disable_dhcp true boolean
+    v4addr=$(grep ^ipv4_address: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/deploycfg|cut -d: -f 2| sed -e 's/ //')
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ^ipv4_netmask: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+    setdebopt netcfg/get_netmask $v4nm string
+    setdebopt netcfg/get_ipaddress $v4addr string
+    setdebopt netcfg/confirm_static true boolean
+    if [ ! -z "$v4gw" ]; then
+        setdebopt netcfg/get_gateway $v4gw string
+    fi
+    namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+    for namesrv in "$namesrvs"; do
+        setdebopt netcfg/get_nameservers $namesrv string
+    done
+elif [ "$vpcfgmeth" = "dhcp" ]; then
+    setdebopt netcfg/disable_dhcp false boolean
+    setdebopt netcfg/confirm_static false boolean
+fi
+rootpass=$(grep ^rootpassword: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //')
+if [ "$rootpass" = null ] || [ -z "$rootpass" ]; then
+    setdebopt passwd/root-login false boolean
+else
+    setdebopt passwd/root-login true boolean
+    setdebopt passwd/root-password-crypted $rootpass string
+fi
+setdebopt time/zone $(grep ^timezone: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //') string
+ntpsrvs=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for ntpsrv in "$ntpsrvs"; do
+    setdebopt clock-setup/ntp true boolean
+    setdebopt clock-setup/ntep-server $ntpsrv string
+done
+#setdebopt console-setup/layoutcode $(grep ^keymap: /etc/confluent/deploycfg|cut -d ' ' -f 2) string
+setdebopt debian-installer/locale $(grep ^locale: /etc/confluent/deploycfg|cut -d ' ' -f 2) select
+domainname=$(grep ^dnsdomain: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+if [ ! -z "$domainname" ] && [ "$domainname" != "null" ]; then
+    setdebopt netcfg/get_domain $domainname string
+fi
+
+
+

confluent_osdeploy/debian/profiles/default/scripts/pre.sh

@@ -0,0 +1,18 @@
+anna-install openssh-server-udeb
+mkdir -p ~/.ssh/
+cat /ssh/*pubkey > ~/.ssh/authorized_keys
+ssh-keygen -A
+mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+apikey=$(cat /etc/confluent/confluent.apikey)
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey%.pub}-cert.pub
+    keyfile=${pubkey%.pub}
+    wget --post-file=$pubkey --header='CONFLUENT_NODENAME: '$nodename --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/sshcert -O $certfile
+    echo HostKey $keyfile >> /etc/ssh/sshd_config
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config
+done
+
+echo sshd:x:939:939::/: >> /etc/passwd
+/usr/sbin/sshd
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.sh

@@ -11,11 +11,10 @@ confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{pr
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
+(
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 chmod 600 /var/log/confluent/confluent-firstboot.log
-tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
-logshowpid=$!
 while ! ping -c 1 $confluent_mgr >& /dev/null; do
 	sleep 1
 done
@@ -37,4 +36,5 @@ curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLU
 systemctl disable firstboot
 rm /etc/systemd/system/firstboot.service
 rm /etc/confluent/firstboot.ran
-kill $logshowpid
+) &
+tail --pid  $! -F /var/log/confluent/confluent-firstboot.log > /dev/console

confluent_osdeploy/el7-diskless/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk

@@ -54,7 +54,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el7-diskless/profiles/default/scripts/imageboot.sh

@@ -15,7 +15,7 @@ export mountsrc=$loopdev
 losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
 if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
     while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
-        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/os/$confluent_profile/pending/rootimg.key)"
         sleep 1
     done
     cipher=$(head -n 1 /tmp/rootimg.key)

confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh

@@ -9,6 +9,7 @@ nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -46,10 +46,15 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el7/profiles/default/scripts/syncfileclient

@@ -3,9 +3,9 @@ import importlib
 import tempfile
 import json
 import os
+import shutil
 import pwd
 import grp
-import shutil
 try:
     from importlib.machinery import SourceFileLoader
     def load_source(mod, path):
@@ -19,6 +19,7 @@ except IOError:
     apiclient = load_source('apiclient', '/etc/confluent/apiclient')
 
 
+
 def partitionhostsline(line):
     comment = ''
     try:
@@ -271,6 +272,7 @@ def synchronize():
                         os.chown(fname, uid, gid)
     finally:
         shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
 
 
 if __name__ == '__main__':

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -25,6 +25,7 @@ get_remote_apikey() {
         fi
     done
 }
+. /lib/dracut-lib.sh
 root=1
 rootok=1
 netroot=confluent
@@ -94,6 +95,41 @@ cd - > /dev/null
 rm -rf $tpmdir
 touch /etc/confluent/confluent.info
 cd /sys/class/net
+touch /etc/confluent/confluent.info
+confluentsrv=$(getarg confluent)
+DIDDHCP=0
+if [ ! -z "$confluentsrv" ]; then
+    for i in *; do
+        ip link set $i up
+    done
+    if [[ "$confluentsrv" = *":"* ]]; then
+        confluenthttpsrv=[$confluentsrv]
+        /usr/libexec/nm-initrd-generator ip=:dhcp6
+    else
+        confluenthttpsrv=$confluentsrv
+        ifname=$(ip -br link|grep LOWER_UP|grep -v UNKNOWN|head -n 1|awk '{print $1}')
+        echo -n "Attempting to use dhcp to bring up $ifname..."
+        dhclient $ifname
+        while ! ip -br addr show dev $ifname | grep \\. > /dev/null; do
+            echo -n "Still waiting for IPv4 address on: "
+            ip -br link show dev $ifname 
+            sleep 1
+        done
+        echo -n "Complete: "
+        ip -br addr show dev $ifname
+        DIDDHCP=1
+    fi
+    myids=uuid=$(cat /sys/devices/virtual/dmi/id/product_uuid)
+    for mac in $(ip -br link|grep -v LOOPBACK|awk '{print $3}'); do
+        myids=$myids"/mac="$mac
+    done
+    myname=$(curl -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
+    if [ ! -z "$myname" ]; then
+        echo NODENAME: $myname > /etc/confluent/confluent.info
+        echo MANAGER: $confluentsrv >> /etc/confluent/confluent.info
+        echo EXTMGRINFO: $confluentsrv'||1' >> /etc/confluent/confluent.info
+    fi
+fi
 echo -n "Scanning for network configuration..."
 while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
     TRIES=$((TRIES + 1))
@@ -123,7 +159,7 @@ while [ $ready = "0" ]; do
         confluent_mgr="[$confluent_mgr]"
     fi
     tmperr=$(mktemp)
-    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
+    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg 2> $tmperr
     if grep 401 $tmperr > /dev/null; then
         confluent_apikey=""
         if [ -n "$lasthdl" ]; then
@@ -147,13 +183,43 @@ if [ -c /dev/tpmrm0 ]; then
     tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
 fi
 umask $oldumask
+mkdir -p /run/NetworkManager/system-connections
+cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[connection]
+EOC
+echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
+echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+type=ethernet
+autoconnect-retries=1
+EOC
+echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+multi-connect=1
+permissions=
+wait-device-timeout=60000
+
+[ethernet]
+mac-address-blacklist=
+
+EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo -n "Attempting to use dhcp to bring up $ifname..."
-    dhclient $ifname
-    echo "Complete:"
-    ip addr show dev $ifname
-else
+    if [ "$DIDDHCP" = "0" ]; then
+        echo -n "Attempting to use dhcp to bring up $ifname..."
+        dhclient $ifname
+        while ! ip -br addr show dev $ifname | grep \\. > /dev/null; do
+            echo -n "Still waiting for IPv4 address on: "
+            ip -br link show dev $ifname 
+            sleep 1
+        done
+        echo -n "Complete: "
+        ip -br addr show dev $ifname
+    fi
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+elif [ "$autoconfigmethod" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
     v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
     v4addr=${v4addr#ipv4_address: }
     v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
@@ -167,28 +233,8 @@ else
     ip addr add dev $ifname $v4addr/$v4nm
     if [ ! -z "$v4gw" ]; then
         ip route add default via $v4gw
-    fi
-    mkdir -p /run/NetworkManager/system-connections
-    cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-[connection]
-EOC
-    echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
-autoconnect-retries=1
-EOC
-    echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-multi-connect=1
-permissions=
-wait-device-timeout=60000
-
-[ethernet]
-mac-address-blacklist=
-
-[ipv4]
-EOC
+    fi    
+    echo '[ipv4]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
     echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
     if [ ! -z "$v4gw" ]; then
         echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -197,7 +243,7 @@ EOC
     nameservers=""
     while read -r entry; do
         if [ $nameserversec = 1 ]; then
-            if [[ $entry == "-"* ]]; then
+            if [[ $entry == "-"*.* ]]; then
                 nameservers="$nameservers"${entry#- }";"
                 continue
             fi
@@ -220,16 +266,69 @@ method=manual
 addr-gen-mode=eui64
 method=auto
 
-[proxy]
 EOC
+elif [ "$auto6configmethod" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+    v6addr=$(grep ^ipv6_address: /etc/confluent/confluent.deploycfg)
+    v6addr=${v6addr#ipv6_address: }
+    v6gw=$(grep ^ipv6_gateway: /etc/confluent/confluent.deploycfg)
+    v6gw=${v6gw#ipv6_gateway: }
+    if [ "$v6gw" = "null" ]; then
+        v6gw=""
+    fi
+    v6nm=$(grep ^ipv6_prefix: /etc/confluent/confluent.deploycfg)
+    v6nm=${v6nm#ipv6_prefix: }
+    echo "Setting up $ifname as static at $v6addr/$v6nm"
+    ip addr add dev $ifname $v6addr/$v6nm
+    
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[ipv4]
+dhcp-timeout=90
+dhcp-vendor-class-identifier=anaconda-Linux
+method=disabled
+
+[ipv6]
+addr-gen-mode=eui64
+method=manual
+may-fail=false
+EOC
+    echo address1=$v6addr/$v6nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v6gw" ]; then
+        ip route add default via $v6gw
+        echo gateway=$v6gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"*:* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
 fi
+echo '[proxy]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
 chmod 600 /run/NetworkManager/system-connections/*.nmconnection
+confluent_websrv=$confluent_mgr
+if [[ $confluent_websrv == *:* ]]; then
+    confluent_websrv="[$confluent_websrv]"
+fi
 echo -n "Initializing ssh..."
 ssh-keygen -A
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}
-    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
+    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_websrv/confluent-api/self/sshcert > $certfile
     if [ -s $certfile ]; then
         echo HostCertificate $certfile >> /etc/ssh/sshd_config
     fi
@@ -246,8 +345,7 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed
         confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
     fi
 done
-confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
 mkdir -p /etc/confluent
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+curl -sf https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
 . /etc/confluent/functions
 source_remote imageboot.sh

confluent_osdeploy/el8-diskless/profiles/default/scripts/add_local_repositories

@@ -7,15 +7,26 @@ import imp
 import sys
 apiclient = imp.load_source('apiclient', '/opt/confluent/bin/apiclient')
 repo = None
+server4 = None
 server = None
+server6 = None
 profile = None
+v4cfg = None
 with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
     lines = dplcfgfile.read().split('\n')
     for line in lines:
         if line.startswith('deploy_server:'):
-           _, server = line.split(' ', 1)
+           _, server4 = line.split(' ', 1)
+        if line.startswith('deploy_server_v6:'):
+           _, server6 = line.split(' ', 1)
         if line.startswith('profile: '):
            _, profile = line.split(' ', 1)
+        if line.startswith('ipv4_method: '):
+           _, v4cfg = line.split(' ', 1)
+if v4cfg == 'static' or v4cfg =='dhcp':
+    server = server4
+if not server:
+    server = '[{}]'.format(server6)
 
 path = '/confluent-public/os/{0}/distribution/'.format(profile)
 clnt = apiclient.HTTPSClient()

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -11,11 +11,10 @@ confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{pr
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
+(
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 chmod 600 /var/log/confluent/confluent-firstboot.log
-tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
-logshowpid=$!
 while ! ping -c 1 $confluent_mgr >& /dev/null; do
 	sleep 1
 done
@@ -37,4 +36,5 @@ curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLU
 systemctl disable firstboot
 rm /etc/systemd/system/firstboot.service
 rm /etc/confluent/firstboot.ran
-kill $logshowpid
+) &
+tail --pid  $! -F /var/log/confluent/confluent-firstboot.log > /dev/console

confluent_osdeploy/el8-diskless/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk

@@ -54,7 +54,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -1,10 +1,14 @@
 . /lib/dracut-lib.sh
+confluent_whost=$confluent_mgr
+if [[ "$confluent_whost" == *:* ]]; then
+    confluent_whost="[$confluent_mgr]"
+fi
 mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
 if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
-    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
+    curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
-    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    confluent_urls="$confluent_urls https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs"
     /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
 fi
 /opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
@@ -12,8 +16,8 @@ loopdev=$(losetup -f)
 export mountsrc=$loopdev
 losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
 if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
-    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
-        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/os/$confluent_profile/pending/rootimg.key)"
         sleep 1
     done
     cipher=$(head -n 1 /tmp/rootimg.key)
@@ -103,15 +107,15 @@ echo 'Host *' >> $sshconf
 echo '    HostbasedAuthentication yes' >> $sshconf
 echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
 cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
 chmod 640 /sysroot/etc/ssh/*_key
 chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
 cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/ update-ca-trust
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
+curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
 mkdir -p /sysroot/opt/confluent/bin
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
+curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
 chmod +x /sysroot/opt/confluent/bin/onboot.sh
 cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
 ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -7,8 +7,15 @@
 
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+if [ "$v4meth" = "null" -o -z "$v4meth" ]; then
+    confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+fi
+if [ -z "$confluent_mgr" ]; then
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+fi
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 [ -e /tmp/confluent.initq ] && return 0
+. /lib/dracut-lib.sh
 udevadm trigger
 udevadm trigger --type=devices --action=add
 udevadm settle
@@ -38,17 +39,97 @@ umask 0077
 mkdir -p /etc/confluent
 echo -n > /etc/confluent/confluent.info
 umask $oum
-cd /sys/class/net
-while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
-    TRIES=$((TRIES + 1))
-    for currif in *; do
-        echo 0 > /proc/sys/net/ipv6/conf/${currif}/autoconf
-        ip link set $currif up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        /usr/libexec/nm-initrd-generator ip=:dhcp
+    else
+        v4addr=$(grep ^ipv4_address: $tcfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4addr=${v4addr%/*}
+        v4gw=$(grep ^ipv4_gateway: $tcfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: $tcfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
+    fi
+    NetworkManager --configure-and-quit=initrd --no-daemon
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    echo 'NODENAME: '$nodename > /etc/confluent/confluent.info
+    for deploysrv in $deploysrvs; do
+        echo 'MANAGER: '$deploysrv >> /etc/confluent/confluent.info
+    done
+    for deployer in $deploysrvs; do
+        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+            cp $passfile /etc/confluent/confluent.apikey
+            confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+            confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+            confluent_profile=${confluent_profile#profile: }
+            mgr=$deployer
+            break
+        fi
+    done
+fi
+cd /sys/class/net
+if ! grep MANAGER: /etc/confluent/confluent.info; then
+    confluentsrv=$(getarg confluent)
+    if [ ! -z "$confluentsrv" ]; then
+        if [[ "$confluentsrv" = *":"* ]]; then
+            confluenthttpsrv=[$confluentsrv]
+            /usr/libexec/nm-initrd-generator ip=:dhcp6
+        else
+            confluenthttpsrv=$confluentsrv
+            /usr/libexec/nm-initrd-generator ip=:dhcp
+        fi
+        NetworkManager --configure-and-quit=initrd --no-daemon
+        myids=uuid=$(cat /sys/devices/virtual/dmi/id/product_uuid)
+        for mac in $(ip -br link|grep -v LOOPBACK|awk '{print $3}'); do
+            myids=$myids"/mac="$mac
+        done
+        myname=$(curl -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
+        if [ ! -z "$myname" ]; then
+            echo NODENAME: $myname > /etc/confluent/confluent.info
+            echo MANAGER: $confluentsrv >> /etc/confluent/confluent.info
+            echo EXTMGRINFO: $confluentsrv'||1' >> /etc/confluent/confluent.info
+        fi
+    fi
+    while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+        TRIES=$((TRIES + 1))
+        for currif in *; do
+            echo 0 > /proc/sys/net/ipv6/conf/${currif}/autoconf
+            ip link set $currif up
+        done
+        /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    done
+    grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+fi
 cd /
-grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
 echo -n "" > /tmp/confluent.initq
 # restart cmdline
 echo -n "" > /etc/cmdline.d/01-confluent.conf
@@ -66,10 +147,15 @@ while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self
 	sleep 10
 done
 ifidx=$(cat /tmp/confluent.ifidx)
-ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-ifname=${ifname%:}
-ifname=${ifname%@*}
-echo $ifname > /tmp/net.ifaces
+if [ ! -z "$ifidx" ]; then
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+    ifname=${ifname%@*}
+    echo $ifname > /tmp/net.ifaces
+else
+    ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
+    ifname=$(cat /tmp/net.ifaces)
+fi
 
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
@@ -84,7 +170,8 @@ v4cfg=${v4cfg#ipv4_method: }
 if [ "$v4cfg" = "static" ] || [ "$v4cfg" = "dhcp" ]; then
     mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
     mgr=${mgr#deploy_server: }
-else
+fi
+if [ -z "$mgr" ]; then
     mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg)
     mgr=${mgr#deploy_server_v6: }
     mgr="[$mgr]"

confluent_osdeploy/el8/profiles/default/kickstart

@@ -59,10 +59,16 @@ v4cfg=${v4cfg#ipv4_method: }
 if [ "$v4cfg" = "static" ] || [ "$v4cfg" = "dhcp" ]; then
     confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server: }
-else
+fi
+if [ -z "$confluent_mgr" ]; then
     confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server_v6: }
-    confluent_mgr="[$confluent_mgr]"
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+        confluent_mgr=${confluent_mgr#deploy_server: }
+    else
+        confluent_mgr="[$confluent_mgr]"
+    fi
 fi
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/preinst.sh
@@ -77,10 +83,16 @@ v4cfg=${v4cfg#ipv4_method: }
 if [ "$v4cfg" = "static" ] || [ "$v4cfg" = "dhcp" ]; then
     confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server: }
-else
+fi
+if [ -z "$confluent_mgr" ]; then
     confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server_v6: }
-    confluent_mgr="[$confluent_mgr]"
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+        confluent_mgr=${confluent_mgr#deploy_server: }
+    else
+        confluent_mgr="[$confluent_mgr]"
+    fi
 fi
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/postinst.sh
 . /tmp/postinst.sh
@@ -104,10 +116,16 @@ v4cfg=${v4cfg#ipv4_method: }
 if [ "$v4cfg" = "static" ] || [ "$v4cfg" = "dhcp" ]; then
     confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server: }
-else
+fi
+if [ -z "$confluent_mgr" ]; then
     confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server_v6: }
-    confluent_mgr="[$confluent_mgr]"
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+        confluent_mgr=${confluent_mgr#deploy_server: }
+    else
+        confluent_mgr="[$confluent_mgr]"
+    fi
 fi
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/postinst.sh
 . /tmp/postinst.sh

confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories

@@ -12,14 +12,25 @@ apiclient = importlib.util.module_from_spec(modspec)
 modspec.loader.exec_module(apiclient)
 repo = None
 server = None
+v4cfg = None
+server4 = None
+server6 = None
 profile = None
 with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
     lines = dplcfgfile.read().split('\n')
     for line in lines:
         if line.startswith('deploy_server:'):
-           _, server = line.split(' ', 1)
+           _, server4 = line.split(' ', 1)
+        if line.startswith('deploy_server_v6:'):
+           _, server6 = line.split(' ', 1)
         if line.startswith('profile: '):
            _, profile = line.split(' ', 1)
+        if line.startswith('ipv4_method: '):
+           _, v4cfg = line.split(' ', 1)
+if v4cfg == 'static' or v4cfg =='dhcp':
+    server = server4
+if not server:
+    server = '[{}]'.format(server6)
 
 path = '/confluent-public/os/{0}/distribution/'.format(profile)
 clnt = apiclient.HTTPSClient()

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -13,20 +13,26 @@ if [ "$v4cfg" = "static" ] || [ "$v4cfg" = "dhcp" ]; then
     confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server: }
     confluent_pingtarget=$confluent_mgr
-else
+fi
+if [ -z "$confluent_mgr" ]; then
     confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg)
     confluent_mgr=${confluent_mgr#deploy_server_v6: }
-    confluent_pingtarget=$confluent_mgr
-    confluent_mgr="[$confluent_mgr]"
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+        confluent_mgr=${confluent_mgr#deploy_server: }
+        confluent_pingtarget=$confluent_mgr
+    else
+        confluent_pingtarget=$confluent_mgr
+        confluent_mgr="[$confluent_mgr]"
+    fi
 fi
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
+(
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 chmod 600 /var/log/confluent/confluent-firstboot.log
-tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
-logshowpid=$!
 while ! ping -c 1 $confluent_pingtarget >& /dev/null; do
 	sleep 1
 done
@@ -49,4 +55,5 @@ curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLU
 systemctl disable firstboot
 rm /etc/systemd/system/firstboot.service
 rm /etc/confluent/firstboot.ran
-kill $logshowpid
+) &
+tail --pid $! -n 0 -F /var/log/confluent/confluent-firstboot.log > /dev/console

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -54,7 +54,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -77,6 +77,7 @@ if [ ! -z "$blargs" ]; then
     echo "bootloader $blargs" > /tmp/grubpw
 fi
 ssh-keygen -A
+rm /etc/ssh/ssh_host_dsa_key*
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$pubkey https://$confluent_mgr/confluent-api/self/sshcert > $certfile

confluent_osdeploy/el8/profiles/default/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -28,7 +28,7 @@ get_remote_apikey() {
 root=1
 rootok=1
 netroot=confluent
-#clear
+echo -ne '\033[H\033[2J\033[3J'
 mkdir -p /etc/ssh
 mkdir -p /var/tmp/
 mkdir -p /var/empty/sshd
@@ -124,7 +124,7 @@ while [ $ready = "0" ]; do
         confluent_mgr="[$confluent_mgr]"
     fi
     tmperr=$(mktemp)
-    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
+    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg 2> $tmperr
     if grep 401 $tmperr > /dev/null; then
         confluent_apikey=""
         if [ -n "$lasthdl" ]; then
@@ -148,13 +148,36 @@ if [ -c /dev/tpmrm0 ]; then
     tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
 fi
 umask $oldumask
+mkdir -p /run/NetworkManager/system-connections
+cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[connection]
+EOC
+echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
+echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+type=ethernet
+autoconnect-retries=1
+EOC
+echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
+cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+multi-connect=1
+permissions=
+wait-device-timeout=60000
+
+[ethernet]
+mac-address-blacklist=
+
+EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 if [ "$autoconfigmethod" = "dhcp" ]; then
     echo -n "Attempting to use dhcp to bring up $ifname..."
     dhclient $ifname
     echo "Complete:"
     ip addr show dev $ifname
-else
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+elif [ "$autoconfigmethod" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
     v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
     v4addr=${v4addr#ipv4_address: }
     v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
@@ -169,27 +192,7 @@ else
     if [ ! -z "$v4gw" ]; then
         ip route add default via $v4gw
     fi
-    mkdir -p /run/NetworkManager/system-connections
-    cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-[connection]
-EOC
-    echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
-autoconnect-retries=1
-EOC
-    echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
-    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-multi-connect=1
-permissions=
-wait-device-timeout=60000
-
-[ethernet]
-mac-address-blacklist=
-
-[ipv4]
-EOC
+    echo '[ipv4]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
     echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
     if [ ! -z "$v4gw" ]; then
         echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -198,7 +201,7 @@ EOC
     nameservers=""
     while read -r entry; do
         if [ $nameserversec = 1 ]; then
-            if [[ $entry == "-"* ]]; then
+            if [[ $entry == "-"*.* ]]; then
                 nameservers="$nameservers"${entry#- }";"
                 continue
             fi
@@ -221,16 +224,69 @@ method=manual
 addr-gen-mode=eui64
 method=auto
 
-[proxy]
 EOC
+elif [ "$auto6configmethod" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+    v6addr=$(grep ^ipv6_address: /etc/confluent/confluent.deploycfg)
+    v6addr=${v6addr#ipv6_address: }
+    v6gw=$(grep ^ipv6_gateway: /etc/confluent/confluent.deploycfg)
+    v6gw=${v6gw#ipv6_gateway: }
+    if [ "$v6gw" = "null" ]; then
+        v6gw=""
+    fi
+    v6nm=$(grep ^ipv6_prefix: /etc/confluent/confluent.deploycfg)
+    v6nm=${v6nm#ipv6_prefix: }
+    echo "Setting up $ifname as static at $v6addr/$v6nm"
+    ip addr add dev $ifname $v6addr/$v6nm
+
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[ipv4]
+dhcp-timeout=90
+dhcp-vendor-class-identifier=anaconda-Linux
+method=disabled
+
+[ipv6]
+addr-gen-mode=eui64
+method=manual
+may-fail=false
+EOC
+    echo address1=$v6addr/$v6nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v6gw" ]; then
+        ip route add default via $v6gw
+        echo gateway=$v6gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"*:* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
 fi
+echo '[proxy]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
 chmod 600 /run/NetworkManager/system-connections/*.nmconnection
+confluent_websrv=$confluent_mgr
+if [[ $confluent_websrv == *:* ]]; then
+    confluent_websrv="[$confluent_websrv]"
+fi
 echo -n "Initializing ssh..."
 ssh-keygen -A
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}
-    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
+    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_websrv/confluent-api/self/sshcert > $certfile
     if [ -s $certfile ]; then
         echo HostCertificate $certfile >> /etc/ssh/sshd_config
     fi
@@ -247,8 +303,7 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed
         confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
     fi
 done
-confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
 mkdir -p /etc/confluent
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+curl -sf https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
 . /etc/confluent/functions
 source_remote imageboot.sh

confluent_osdeploy/el9-diskless/profiles/default/scripts/add_local_repositories

@@ -12,14 +12,25 @@ apiclient = importlib.util.module_from_spec(modspec)
 modspec.loader.exec_module(apiclient)
 repo = None
 server = None
+v4cfg = None
+server4 = None
+server6 = None
 profile = None
 with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
     lines = dplcfgfile.read().split('\n')
     for line in lines:
         if line.startswith('deploy_server:'):
-           _, server = line.split(' ', 1)
+           _, server4 = line.split(' ', 1)
+        if line.startswith('deploy_server_v6:'):
+           _, server6 = line.split(' ', 1)
         if line.startswith('profile: '):
            _, profile = line.split(' ', 1)
+        if line.startswith('ipv4_method: '):
+           _, v4cfg = line.split(' ', 1)
+if v4cfg == 'static' or v4cfg =='dhcp':
+    server = server4
+if not server:
+    server = '[{}]'.format(server6)
 
 path = '/confluent-public/os/{0}/distribution/'.format(profile)
 clnt = apiclient.HTTPSClient()

confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh

@@ -11,11 +11,10 @@ confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{pr
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
+(
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 chmod 600 /var/log/confluent/confluent-firstboot.log
-tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
-logshowpid=$!
 while ! ping -c 1 $confluent_mgr >& /dev/null; do
 	sleep 1
 done
@@ -37,4 +36,5 @@ curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLU
 systemctl disable firstboot
 rm /etc/systemd/system/firstboot.service
 rm /etc/confluent/firstboot.ran
-kill $logshowpid
+) &
+tail --pid  $! -F /var/log/confluent/confluent-firstboot.log > /dev/console

confluent_osdeploy/el9-diskless/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk

@@ -54,7 +54,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -1,10 +1,14 @@
 . /lib/dracut-lib.sh
+confluent_whost=$confluent_mgr
+if [[ "$confluent_whost" == *:* ]]; then
+    confluent_whost="[$confluent_mgr]"
+fi
 mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
 if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
     mount -t tmpfs untethered /mnt/remoteimg
-    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
+    curl https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
 else
-    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    confluent_urls="$confluent_urls https://$confluent_whost/confluent-public/os/$confluent_profile/rootimg.sfs"
     /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
 fi
 /opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
@@ -12,8 +16,8 @@ loopdev=$(losetup -f)
 export mountsrc=$loopdev
 losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
 if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
-    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
-        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/os/$confluent_profile/pending/rootimg.key)"
         sleep 1
     done
     cipher=$(head -n 1 /tmp/rootimg.key)
@@ -103,15 +107,15 @@ echo 'Host *' >> $sshconf
 echo '    HostbasedAuthentication yes' >> $sshconf
 echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_whost/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
 cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
 chmod 640 /sysroot/etc/ssh/*_key
 chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
 cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
 chroot /sysroot/ update-ca-trust
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
+curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
 mkdir -p /sysroot/opt/confluent/bin
-curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
+curl -sf https://$confluent_whost/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
 chmod +x /sysroot/opt/confluent/bin/onboot.sh
 cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
 ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service

confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh

@@ -7,8 +7,15 @@
 
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+if [ "$v4meth" = "null" -o -z "$v4meth" ]; then
+    confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+fi
+if [ -z "$confluent_mgr" ]; then
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+fi
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -1,13 +1,82 @@
 #!/bin/sh
 # copernicus is hard coded, easier to script a fake sysfs
+if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
+    chvt 1
+    sleep 1
+    chvt 2
+    /etc/init.d/ESXShell start
+fi
 uuid=$(vsish -e get /hardware/bios/dmiInfo|grep -A15 UUID|sed -e 's/.*://'|sed -e ':a;N;$!ba;s/\n//g' | sed -e 's/ *0x//g')
 uuid=${uuid:0:8}-${uuid:8:4}-${uuid:12:4}-${uuid:16:4}-${uuid:20:12}
+kargs=$(vsish -e get /system/bootCmdLine|grep "command line:")
+vlanid=""
+for karg in $kargs; do
+    if [ ${karg%=*} = "vlanid" ]; then
+        vlanid=${karg/*=}
+    fi
+done
+if [ ! -z "$vlanid" ]; then
+    echo $vlanid > /tmp/confluent.vlanid
+    localcli network  vswitch standard portgroup set -p "Management Network" -v $vlanid
+fi
 mkdir -p /sys/devices/virtual/dmi/id/
 echo $uuid > /sys/devices/virtual/dmi/id/product_uuid
 mkdir -p /etc/confluent
 localcli network firewall unload
 touch /etc/confluent/confluent.info
 begin=$(date +%s)
+localcli system secpolicy domain set -n appDom -l disabled
+personality=''
+for usb in /dev/disks/mpx.vmhba*; do 
+    if mdir -i $usb |grep CNFLNT_IDNT > /dev/null; then
+	personality=$usb
+	break
+    fi
+done
+if [ ! -z "$personality" ]; then
+    mkdir -p /tmp/confluentident
+    mcopy -i $personality ::* /tmp/confluentident/
+fi
+touch /etc/confluent/confluent.apikey
+if [ -e /tmp/confluentident/cnflnt.yml ]; then
+    cat /tls/*.pem >> /etc/confluent/ca.pem
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' /tmp/confluentident/cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: /tmp/confluentident/cnflnt.yml|sed -e 's/nodename: //')
+    echo 'NODENAME: '$nodename > /etc/confluent/confluent.info
+    for deploysrv in $deploysrvs; do
+        echo 'MANAGER: '$deploysrv >> /etc/confluent/confluent.info
+    done
+    tcfg=$(mktemp)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' /tmp/confluentident/cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "static" ]; then
+        v4addr=$(grep ^ipv4_address: $tcfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4addr=${v4addr%/*}
+        v4gw=$(grep ^ipv4_gateway: $tcfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: $tcfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        localcli network ip interface ipv4 set -i vmk0 -I $v4addr -N $v4nm -g $v4gw -t static
+        localcli network ip route ipv4 add -n default -g $v4gw
+    fi
+    hmackeyfile=$(mktemp)
+    echo -n $(grep ^apitoken: /tmp/confluentident/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    passfile=$(mktemp)
+    passcrypt=$(mktemp)
+    hmacfile=$(mktemp)
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    echo -n 'Registering new API key with deployment server: '
+    /opt/confluent/bin/apiclient -p $hmacfile /confluent-api/self/registerapikey $passcrypt
+    echo
+    cp $passfile /etc/confluent/confluent.apikey
+fi
 while ! grep NODENAME /etc/confluent/confluent.info; do
     echo "Searching for deployment service on local network..."
     /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
@@ -25,20 +94,18 @@ while ! grep NODENAME /etc/confluent/confluent.info; do
     fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
-mgr=$(grep MANAGER: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
-cp /opt/confluent/bin/clortho /clortho
-/clortho $node $mgr > /etc/confluent/confluent.apikey
+mgr=$(python /opt/confluent/bin/apiclient -c)
 APIKEY=$(cat /etc/confluent/confluent.apikey)
+
+if [ -z "$APIKEY" ]; then
+    cp /opt/confluent/bin/clortho /clortho
+    (/clortho $node $mgr || /opt/confluent/bin/clortho $node $mgr) > /etc/confluent/confluent.apikey
+    APIKEY=$(cat /etc/confluent/confluent.apikey)
+fi
 while [ -z "$APIKEY" ]; do
-    if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
-        chvt 1
-        sleep 1
-        chvt 2
-        /etc/init.d/ESXShell start
-    fi
     echo "Deployment server failed to grant node deployment token, retrying token retrieval"
     echo "A debug session has been made available on Alt-F1"
-    /clortho $node $mgr > /etc/confluent/confluent.apikey
+    (/clortho $node $mgr || /opt/confluent/bin/clortho $node $mgr) > /etc/confluent/confluent.apikey
     APIKEY=$(cat /etc/confluent/confluent.apikey)
 done
 cat /tls/*.pem > /etc/confluent/ca.pem

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 import re
 import subprocess
-
+import json
 uplinkmatch = re.compile('^\s*Uplinks:\s*(.*)')
 nodename = None
 for inf in open('/etc/confluent/confluent.info', 'r').read().split('\n'):
@@ -33,6 +33,16 @@ for info in vswinfo.split('\n'):
     upinfo = uplinkmatch.match(info)
     if upinfo:
         vmnic = upinfo.group(1)
+try:
+    with open('/tmp/confluentident/cnflnt.jsn') as identin:
+        identcfg = json.load(identin)
+        ncfg = identcfg['net_cfgs'][0]
+        cfg['ipv4_method'] = ncfg['ipv4_method']
+        cfg['ipv4_address'] = ncfg['ipv4_address'].split('/')[0]
+        cfg['ipv4_netmask'] = ncfg['ipv4_netmask']
+        cfg['ipv4_gateway'] = ncfg['ipv4_gateway']
+except Exception:
+    pass
 netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
 if vmnic:
     netline += ' --device={0}'.format(vmnic)
@@ -42,5 +52,12 @@ if cfg['ipv4_method'] == 'static':
         netline += ' --gateway={0}'.format(cfg['ipv4_gateway'])
     if cfg['nameservers']:
         netline += ' --nameserver={0}'.format(cfg['nameservers'])
+vlanid = None
+try:
+    with open('/tmp/confluent.vlanid', 'r') as vlanin:
+        vlanid = vlanin.read()
+except Exception:
+    pass
+if vlanid:
+    netline += ' --vlanid={0}'.format(vlanid)
 print(netline)
-    

confluent_osdeploy/genesis/profiles/default/scripts/functions

@@ -1,10 +1,4 @@
 #!/bin/bash
-function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
-        return 0
-    fi
-    return 1
-}
 
 function confluentpython() {
     if [ -x /usr/libexec/platform-python ]; then
@@ -22,29 +16,9 @@ function set_confluent_vars() {
     if [ -z "$nodename" ]; then
         nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
     fi
-    if [[ "$confluent_mgr" == *"%"* ]]; then
-        confluent_mgr=""
-    fi
-    if [ -z "$confluent_mgr" ]; then
-        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
-        if ! test_mgr $confluent_mgr; then
-            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
-            if [[ "$confluent_mgr" = *":"* ]]; then
-                confluent_mgr="[$confluent_mgr]"
-            fi
-        fi
-        if ! test_mgr $confluent_mgr; then
-            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
-            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
-            for confluent_mgr in $BESTMGRS $OKMGRS; do
-                if [[ $confluent_mgr == *":"* ]]; then
-                    confluent_mgr="[$confluent_mgr]"
-                fi
-                if test_mgr $confluent_mgr; then
-                    break
-                fi
-            done
-        fi
+    confluent_mgr=$(confluentpython /opt/confluent/bin/apiclient -c)
+    if [[ "$confluent_mgr" = *":"* ]]; then
+        confluent_mgr="[$confluent_mgr]"
     fi
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
@@ -52,13 +26,13 @@ function set_confluent_vars() {
 }
 
 fetch_remote() {
-    curlargs=""
-    if [ -f /etc/confluent/ca.pem ]; then
-        curlargs=" --cacert /etc/confluent/ca.pem"
-    fi
     set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    confluentpython $apiclient /confluent-public/os/$confluent_profile/scripts/$1 -o $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -156,16 +130,13 @@ run_remote() {
 run_remote_python() {
     echo
     set_confluent_vars
-    if [ -f /etc/confluent/ca.pem ]; then
-        curlargs=" --cacert /etc/confluent/ca.pem"
-    fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    confluentpython /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/$1 -o $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk

@@ -46,10 +46,15 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/suse15-diskless/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/suse15-diskless/profiles/default/scripts/onboot.sh

@@ -9,6 +9,7 @@ nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
 mkdir -p /var/log/confluent

confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1,286 @@
+#!/usr/bin/python3
+import subprocess
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/suse15-diskless/profiles/default/syncfiles

@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group

confluent_osdeploy/suse15/initramfs/opt/confluent/bin/suseagent

@@ -81,6 +81,7 @@ fi
 
 tz=$(grep timezone: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 echo "<timezone>${tz}</timezone>" > /tmp/timezone
+echo "${tz}" > /tmp/timezone.txt
 autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
 autoconfigmethod=${autoconfigmethod#ipv4_method: }
 if [ "$autoconfigmethod" = "dhcp" ]; then

confluent_osdeploy/suse15/profiles/hpc/autoyast.leap

@@ -0,0 +1,132 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<!--
+This autoyast file will be processed by pre.sh before applying. See pre.sh for
+info on modifying its behavior, and also search and replace '%%' to remove
+dynamic behavior and replace with static configuration.
+--> 
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+    <timezone>
+      <hwclock>UTC</hwclock>
+      <timezone>%%TIMEZONE%%</timezone>
+    </timezone>
+    <general>
+      <self_update config:type="boolean">false</self_update>
+      <mode>
+	      <confirm config:type="boolean">false</confirm>
+      </mode>
+    </general>
+    <!--INSERT:/tmp/bootloader.xml-->
+    <software>
+      <patterns config:type="list">
+        <pattern>base</pattern>
+      </patterns>
+      <packages config:type="list">
+	<package>openssh</package>
+	<package>iputils</package>
+        <package>python3</package>
+        <package>openssl</package>
+        <package>chrony</package>
+        <package>rsync</package>
+        <package>screen</package>
+        <package>vim</package>
+        <package>binutils</package>
+        <package>pciutils</package>
+        <package>usbutils</package>
+        <package>nfs-client</package>
+        <package>ethtool</package>
+      </packages>
+    </software>
+  <partitioning config:type="list">
+	  <drive>
+		  <device>%%INSTDISK%%</device>
+		  <initialize config:type="boolean">true</initialize>
+		  <use>all</use>
+                  <partitions config:type="list">
+                         <partition>
+                                 <filesystem config:type="symbol">xfs</filesystem>
+                                 <mount>/</mount>
+                                 <size>max</size>
+                         </partition>
+                         <partition>
+                                 <mount>swap</mount>
+                                 <size>auto</size>
+                         </partition>
+                         <partition>
+                                 <mount>/boot</mount>
+                                 <size>500M</size>
+                         </partition>
+                 </partitions>
+          </drive>
+ </partitioning>
+    <users config:type="list">
+      <user>
+        <username>root</username>
+        <user_password>%%ROOTPASSWORD%%</user_password>
+        <encrypted config:type="boolean">true</encrypted>
+	<!--INSERT:/tmp/rootkeys.xml-->
+      </user>
+    </users>
+    <networking>
+	    <dns>
+		    <hostname>%%NODENAME%%</hostname>
+            </dns>
+	    <keep_install_network config:type="boolean">true</keep_install_network>
+    </networking>
+    <services-manager>
+	    <services>
+		    <enable config:type="list">
+			    <service>sshd</service>
+			</enable>
+		</services>
+	</services-manager>
+    <scripts>
+         <pre-scripts config:type="list">
+        <script>
+          <filename>preinstall.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh
+]]>
+          </source>
+        </script>
+      </pre-scripts>
+         <chroot-scripts config:type="list">
+        <script>
+          <filename>chroot.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/prechroot.sh
+. /tmp/prechroot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /mnt/etc/confluent/firstboot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /mnt/etc/confluent/post.sh
+chmod +x /mnt/etc/confluent/firstboot.sh
+chmod +x /mnt/etc/confluent/post.sh
+cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
+]]>
+          </source>
+        </script>
+        <script>
+          <chrooted config:type="boolean">true</chrooted>
+          <filename>post.sh</filename>
+          <location>file:///etc/confluent/post.sh</location>
+        </script>
+      </chroot-scripts>
+      <init-scripts config:type="list">
+        <script>
+          <filename>firstboot.sh</filename>
+          <location>file:///etc/confluent/firstboot.sh</location>
+        </script>
+     </init-scripts> 
+
+    </scripts>
+</profile>

confluent_osdeploy/suse15/profiles/hpc/autoyast.sle

@@ -8,7 +8,7 @@ dynamic behavior and replace with static configuration.
 <profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
     <timezone>
       <hwclock>UTC</hwclock>
-      <!--INSERT:/tmp/timezone-->
+      <timezone>%%TIMEZONE%%</timezone>
     </timezone>
     <general>
       <self_update config:type="boolean">false</self_update>
@@ -16,7 +16,6 @@ dynamic behavior and replace with static configuration.
 	      <confirm config:type="boolean">false</confirm>
       </mode>
     </general>
-    %%IFSLE%%
     <add-on>
       <add_on_products config:type="list">
         <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
@@ -27,14 +26,11 @@ dynamic behavior and replace with static configuration.
 	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
       </add_on_products>
     </add-on>
-    %%ENDIFSLE%%
     <!--INSERT:/tmp/bootloader.xml-->
     <software>
-       %%IFSLE%%
        <products config:type="list">
           <product>SLE_HPC</product>
       </products>
-       %%ENDIFSLE%%
       <patterns config:type="list">
         <pattern>base</pattern>
       </patterns>

confluent_osdeploy/suse15/profiles/hpc/initprofile.sh

@@ -17,7 +17,7 @@ ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
 mkdir -p $2/boot/efi/boot && \
 ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
 if [[ $profile =~ ^sle.* ]]; then
-	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
+	ln -s autoyast.sle $2/autoyast
 else
-	sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast
+	ln -s autoyast.leap $2/autoyast
 fi

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -20,7 +20,8 @@ confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 . /etc/confluent/functions
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
+GIVUP=$(($(date +%s) + 60))
+while (! ping -c 1 $confluent_mgr >& /dev/null) && [ $(date +%s) -lt $GIVUP ]; do
 	sleep 1
 done
 

confluent_osdeploy/suse15/profiles/hpc/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk

@@ -46,10 +46,15 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/suse15/profiles/hpc/scripts/pre.sh

@@ -6,6 +6,7 @@
 exec >> /tmp/confluent-pre.log
 exec 2>> /tmp/confluent-pre.log
 chmod 600 /tmp/confluent-pre.log
+cat /tls/*.pem > /etc/confluent/ca.pem
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 rootpw=$(grep rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
 if [ "$rootpw" = "null" ]; then
@@ -38,20 +39,25 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
     echo '</ntp_servers></ntp-client>' >> /tmp/ntp.cfg
     ntpcfg=$(paste -sd '' /tmp/ntp.cfg)
 fi
-mdadm --assemble --scan
-run_remote_python getinstalldisk
-if grep ^md /tmp/installdisk > /dev/null; then
-   for md in /dev/disk/*/*; do
-	rmd=$(readlink $md)
-	if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then
-	    echo ${md#/dev/} > /tmp/installdisk
-	fi
-   done
+export confluent_mgr confluent_profile nodename
+run_remote pre.custom
+run_remote_parts pre.d
+if [ ! -e /tmp/installdisk ]; then
+    mdadm --assemble --scan
+    run_remote_python getinstalldisk
+    if grep ^md /tmp/installdisk > /dev/null; then
+    for md in /dev/disk/*/*; do
+        rmd=$(readlink $md)
+        if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then
+            echo ${md#/dev/} > /tmp/installdisk
+        fi
+    done
+    fi
 fi
 sed -e s'!'%%INSTDISK%%'!'/dev/$(cat /tmp/installdisk)'!' -e s'!'%%NODENAME%%'!'$nodename'!' -e 's!<networking\(.*\)>!'"$ntpcfg"'<networking\1>!' -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml
 if grep append /tmp/bootloader.xml > /dev/null; then
     sed -i 's@</general>@</general>'"$(tr -d '\n' < /tmp/bootloader.xml)"'@' /tmp/profile/modified.xml
 fi
 sed -i 's#root</username>#root</username>'"$(tr -d '\n' < /tmp/rootkeys.xml)"'#' /tmp/profile/modified.xml
-sed -i 's@/hwclock>@/hwclock>'"$(tr -d '\n' < /tmp/timezone)"'@' /tmp/profile/modified.xml
+sed -i 's@%%TIMEZONE%%@'"$(cat /tmp/timezone.txt)"'@' /tmp/profile/modified.xml
 sed -i 's@<media_url/>@'"$(tr -d '\n' < /tmp/pkgurl)"'@' /tmp/profile/modified.xml

confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/suse15/profiles/hpc/syncfiles

@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group

confluent_osdeploy/suse15/profiles/server/autoyast.leap

@@ -0,0 +1,132 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<!--
+This autoyast file will be processed by pre.sh before applying. See pre.sh for
+info on modifying its behavior, and also search and replace '%%' to remove
+dynamic behavior and replace with static configuration.
+--> 
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+    <timezone>
+      <hwclock>UTC</hwclock>
+      <timezone>%%TIMEZONE%%</timezone>
+    </timezone>
+    <general>
+      <self_update config:type="boolean">false</self_update>
+      <mode>
+	      <confirm config:type="boolean">false</confirm>
+      </mode>
+    </general>
+    <!--INSERT:/tmp/bootloader.xml-->
+    <software>
+      <patterns config:type="list">
+        <pattern>base</pattern>
+      </patterns>
+      <packages config:type="list">
+	<package>openssh</package>
+	<package>iputils</package>
+        <package>python3</package>
+        <package>openssl</package>
+        <package>chrony</package>
+        <package>rsync</package>
+        <package>screen</package>
+        <package>vim</package>
+        <package>binutils</package>
+        <package>pciutils</package>
+        <package>usbutils</package>
+        <package>nfs-client</package>
+        <package>ethtool</package>
+      </packages>
+    </software>
+  <partitioning config:type="list">
+	  <drive>
+		  <device>%%INSTDISK%%</device>
+		  <initialize config:type="boolean">true</initialize>
+		  <use>all</use>
+                  <partitions config:type="list">
+                         <partition>
+                                 <filesystem config:type="symbol">xfs</filesystem>
+                                 <mount>/</mount>
+                                 <size>max</size>
+                         </partition>
+                         <partition>
+                                 <mount>swap</mount>
+                                 <size>auto</size>
+                         </partition>
+                         <partition>
+                                 <mount>/boot</mount>
+                                 <size>500M</size>
+                         </partition>
+                 </partitions>
+          </drive>
+ </partitioning>
+    <users config:type="list">
+      <user>
+        <username>root</username>
+        <user_password>%%ROOTPASSWORD%%</user_password>
+        <encrypted config:type="boolean">true</encrypted>
+	<!--INSERT:/tmp/rootkeys.xml-->
+      </user>
+    </users>
+    <networking>
+	    <dns>
+		    <hostname>%%NODENAME%%</hostname>
+            </dns>
+	    <keep_install_network config:type="boolean">true</keep_install_network>
+    </networking>
+    <services-manager>
+	    <services>
+		    <enable config:type="list">
+			    <service>sshd</service>
+			</enable>
+		</services>
+	</services-manager>
+    <scripts>
+         <pre-scripts config:type="list">
+        <script>
+          <filename>preinstall.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh
+]]>
+          </source>
+        </script>
+      </pre-scripts>
+         <chroot-scripts config:type="list">
+        <script>
+          <filename>chroot.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/prechroot.sh
+. /tmp/prechroot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /mnt/etc/confluent/firstboot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /mnt/etc/confluent/post.sh
+chmod +x /mnt/etc/confluent/firstboot.sh
+chmod +x /mnt/etc/confluent/post.sh
+cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
+]]>
+          </source>
+        </script>
+        <script>
+          <chrooted config:type="boolean">true</chrooted>
+          <filename>post.sh</filename>
+          <location>file:///etc/confluent/post.sh</location>
+        </script>
+      </chroot-scripts>
+      <init-scripts config:type="list">
+        <script>
+          <filename>firstboot.sh</filename>
+          <location>file:///etc/confluent/firstboot.sh</location>
+        </script>
+     </init-scripts> 
+
+    </scripts>
+</profile>

confluent_osdeploy/suse15/profiles/server/autoyast.sle

@@ -8,7 +8,7 @@ dynamic behavior and replace with static configuration.
 <profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
     <timezone>
       <hwclock>UTC</hwclock>
-      <!--INSERT:/tmp/timezone-->
+      <timezone>%%TIMEZONE%%</timezone>
     </timezone>
     <general>
       <self_update config:type="boolean">false</self_update>
@@ -16,7 +16,6 @@ dynamic behavior and replace with static configuration.
 	      <confirm config:type="boolean">false</confirm>
       </mode>
     </general>
-    %%IFSLE%%
     <add-on>
       <add_on_products config:type="list">
         <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
@@ -26,14 +25,11 @@ dynamic behavior and replace with static configuration.
 	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
       </add_on_products>
     </add-on>
-    %%ENDIFSLE%%
     <!--INSERT:/tmp/bootloader.xml-->
     <software>
-       %%IFSLE%%
        <products config:type="list">
           <product>SLES</product>
       </products>
-       %%ENDIFSLE%%
       <patterns config:type="list">
         <pattern>base</pattern>
       </patterns>

confluent_osdeploy/suse15/profiles/server/initprofile.sh

@@ -17,7 +17,7 @@ ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
 mkdir -p $2/boot/efi/boot && \
 ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
 if [[ $profile =~ ^sle.* ]]; then
-	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
+	ln -s autoyast.sle $2/autoyast
 else
-	sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast
+	ln -s autoyast.leap $2/autoyast
 fi

confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh

@@ -20,7 +20,8 @@ confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 . /etc/confluent/functions
-while ! ping -c 1 $confluent_mgr >& /dev/null; do
+GIVUP=$(($(date +%s) + 60))
+while (! ping -c 1 $confluent_mgr >& /dev/null) && [ $(date +%s) -lt $GIVUP ]; do
 	sleep 1
 done
 

confluent_osdeploy/suse15/profiles/server/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk

@@ -46,10 +46,15 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/suse15/profiles/server/scripts/pre.sh

@@ -6,6 +6,7 @@
 exec >> /tmp/confluent-pre.log
 exec 2>> /tmp/confluent-pre.log
 chmod 600 /tmp/confluent-pre.log
+cat /tls/*.pem > /etc/confluent/ca.pem
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 rootpw=$(grep rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
 if [ "$rootpw" = "null" ]; then
@@ -38,20 +39,25 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
     echo '</ntp_servers></ntp-client>' >> /tmp/ntp.cfg
     ntpcfg=$(paste -sd '' /tmp/ntp.cfg)
 fi
-mdadm --assemble --scan
-run_remote_python getinstalldisk
-if grep ^md /tmp/installdisk > /dev/null; then
-   for md in /dev/disk/*/*; do
-	rmd=$(readlink $md)
-	if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then
-	    echo ${md#/dev/} > /tmp/installdisk
-	fi
-   done
+export confluent_mgr confluent_profile nodename
+run_remote pre.custom
+run_remote_parts pre.d
+if [ ! -e /tmp/installdisk ]; then
+    mdadm --assemble --scan
+    run_remote_python getinstalldisk
+    if grep ^md /tmp/installdisk > /dev/null; then
+    for md in /dev/disk/*/*; do
+        rmd=$(readlink $md)
+        if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then
+            echo ${md#/dev/} > /tmp/installdisk
+        fi
+    done
+    fi
 fi
 sed -e s'!'%%INSTDISK%%'!'/dev/$(cat /tmp/installdisk)'!' -e s'!'%%NODENAME%%'!'$nodename'!' -e 's!<networking\(.*\)>!'"$ntpcfg"'<networking\1>!' -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml
 if grep append /tmp/bootloader.xml > /dev/null; then
     sed -i 's@</general>@</general>'"$(tr -d '\n' < /tmp/bootloader.xml)"'@' /tmp/profile/modified.xml
 fi
 sed -i 's#root</username>#root</username>'"$(tr -d '\n' < /tmp/rootkeys.xml)"'#' /tmp/profile/modified.xml
-sed -i 's@/hwclock>@/hwclock>'"$(tr -d '\n' < /tmp/timezone)"'@' /tmp/profile/modified.xml
+sed -i 's@%%TIMEZONE%%@'"$(cat /tmp/timezone.txt)"'@' /tmp/profile/modified.xml
 sed -i 's@<media_url/>@'"$(tr -d '\n' < /tmp/pkgurl)"'@' /tmp/profile/modified.xml

confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/suse15/profiles/server/syncfiles

@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh

@@ -21,7 +21,7 @@ export mountsrc=$loopdev
 losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
 if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
     while ! curl -sf -H "CONFLUENT_NODENAME: $confluent_nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
-        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/os/$confluent_profile/pending/rootimg.key)"
         sleep 1
     done
     cipher=$(head -n 1 /tmp/rootimg.key)

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk

@@ -46,10 +46,15 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions

@@ -1,6 +1,10 @@
 #!/bin/bash
 function test_mgr() {
-    if curl -s https://${1}/confluent-api/ > /dev/null; then
+    whost=$1
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    if curl -s https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -58,7 +62,11 @@ fetch_remote() {
     fi
     set_confluent_vars
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -165,7 +173,11 @@ run_remote_python() {
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
-    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]]; then
+        whost="[$whost]"
+    fi
+    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk

@@ -46,10 +46,15 @@ class DiskInfo(object):
                 self.driver = v.replace('"', '')
         if not self.driver and 'imsm' not in self.mdcontainer:
             raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/python3
+import subprocess
 import importlib
 import tempfile
 import json
@@ -211,7 +212,20 @@ def synchronize():
     appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
             lastrsp = ''

confluent_osdeploy/utils/autocons.c

@@ -68,16 +68,16 @@ int main(int argc, char* argv[]) {
     }
     if (currspeed == SPEED9600) {
         cspeed = B9600;
-        strcpy(offset, ",9600");
+        strncpy(offset, ",9600", 6);
     } else if (currspeed == SPEED19200) {
         cspeed = B19200;
-        strcpy(offset, ",19200");
+        strncpy(offset, ",19200", 7);
     } else if (currspeed == SPEED57600) {
         cspeed = B57600;
-        strcpy(offset, ",57600");
+        strncpy(offset, ",57600", 7);
     } else if (currspeed == SPEED115200) {
         cspeed = B115200;
-        strcpy(offset, ",115200");
+        strncpy(offset, ",115200", 8);
     } else {
         exit(0);
     }
@@ -86,6 +86,7 @@ int main(int argc, char* argv[]) {
         cfsetospeed(&tty, cspeed);
         cfsetispeed(&tty, cspeed);
     }
+    buff[127] = 0;
     printf("%s\n", buff);
     tcgetattr(ttyf, &tty2);
     cfmakeraw(&tty2);

confluent_osdeploy/utils/clortho.c

@@ -90,6 +90,7 @@ int getpasshmac(int argc, char* argv[]) {
     tmps = genpasswd(16);
     memcpy(buffer, "$5$", 3);
     memcpy(buffer + 3, tmps, 16);
+    free(tmps);
     buffer[19] = 0;
     fwrite(passwd, 1, 48, outfile);
     fclose(outfile);
@@ -105,6 +106,7 @@ int getpasshmac(int argc, char* argv[]) {
     free(hmac64);
     free(passwd);
     free(buffer);
+    return 0;
 }
 
 int main(int argc, char* argv[]) {