diff --git a/confluent_client/doc/man/imgutil.ronn b/confluent_client/doc/man/imgutil.ronn index 2705cefc..ff014ec1 100644 --- a/confluent_client/doc/man/imgutil.ronn +++ b/confluent_client/doc/man/imgutil.ronn @@ -3,6 +3,7 @@ ## SYNOPSIS `imgutil` `build` [] +`imgutil` `build` `--containerfile` [] `imgutil` `exec` [] [...] `imgutil` `unpack` `imgutil` `pack` [] @@ -16,6 +17,12 @@ * `build`: Build a new diskless image from scratch in the specified scratch directory. + With a container image as `-s` source, the image is extracted to the + scratch directory instead, and completed with the confluent initramfs + configuration, the package list and a regenerated initramfs. This + builds images made elsewhere, for example in CI. With + `--containerfile`, a Containerfile to start such an image from is + written instead. * `exec`: Start the specified scratch directory as a container and optionally run a command inside it. @@ -43,8 +50,38 @@ * `-a`, `--addpackagelist` : A list of additional packages to include. May be specified multiple times. -* `-s`, `--source` : - Directory to pull installation from, typically a subdirectory of `/var/lib/confluent/distributions`. By default, the repositories for the build system are used. For Ubuntu, this is not supported; the build system repositories are always used. +* `-s`, `--source` |: + Directory to pull installation from, typically a subdirectory of `/var/lib/confluent/distributions`. By default, the repositories for the build system are used. For Ubuntu, a distribution directory is not supported, the build system repositories are always used. + + A container image made elsewhere, for example in CI, can be the source + instead: a SIF file as built by Apptainer or SingularityCE, an OCI or + Docker archive file as written by `podman save`, an OCI layout + directory with its `oci-layout` file, or a podman reference with its + transport, for example `docker://registry.example.com/hpc/compute:latest` + or `containers-storage:localhost/compute` for a local image. Local + images are looked up in the podman storage of the user running + imgutil, normally root. Any other image is pulled or loaded into a + temporary storage next to the scratch directory, which is removed + afterwards, so the images and tags of the system stay as they are. A + path containing a colon cannot be read by podman. A SIF file is + extracted from its squashfs partition, which needs + squashfs-tools 4.4 or newer but not podman. + + Any other directory or name is a distribution as above. The scratch + directory must be empty or not exist. Missing packages are installed + from the repositories of the image, so an image that already has every + package builds without network access. A package list entry with a + version, such as `curl=8.5.0-2ubuntu10`, always runs the package + manager and needs the repositories. Units that container images mask + are unmasked and `/usr/sbin/policy-rc.d` is removed, so the image boots + like an installed system. Package manager settings of the image are + kept, for example container base images leaving out documentation. The + architecture is detected from the image. With `--arch`, the matching + variant of an image for several architectures is used. `-r` is not supported, as the + image brings its own repositories. EL9, Ubuntu 24.04 and SUSE 16 or + later images are supported. With `--selinux`, the full SELinux context + of every file, including the SELinux user, is set from the policy of + the image. * `--arch` : Target architecture to build for (`x86_64` or `aarch64`). For Ubuntu, the @@ -71,8 +108,30 @@ * `-v`, `--volume` : Directory to make available in the build environment. `-v /` will cause it to be mounted in image as `/run/external/`. `-v /:/run/root` will override the target to be `/run/root`. Something like `/var/lib/repository:-` will cause it to mount to the identical path inside the image. May be specified multiple times. +* `--containerfile`: + Write a Containerfile instead of building. It installs every package + `build` checks for, generated from the same package lists, so building + from the image needs no network access. Add site specific content and + build it in CI. The OS is that of the `-s` distribution or of the build + system, as for a build, and `--arch` names the platform, so the image + can be built on another architecture with emulation. `-p`, `-a` and + `--selinux` change the packages. Pass the same options to `build` + later, as the command shown in the file does. `-r` is not supported, + as the image installs from the repositories of its base image. EL9, + Ubuntu 24.04 and SUSE 16 or later are supported. + +* `--apptainer`: + With `--containerfile`, write an Apptainer definition file for building + a SIF instead. + +* `--from` : + With `--containerfile`, the base image to use instead of the default for + the OS (AlmaLinux for EL, the official Ubuntu image, openSUSE Leap for + SUSE). + * : - Directory to build new diskless root in. + Directory to build new diskless root in. With `--containerfile`, the file + to write, or `-` for standard output. ## EXEC OPTIONS @@ -130,6 +189,34 @@ Build an aarch64 Ubuntu diskless image on an amd64 Ubuntu host: imgutil build --arch aarch64 /tmp/myimage +Build from an image made from a Dockerfile and pack it: + + podman build -t localhost/myimage . + imgutil build -s containers-storage:localhost/myimage /tmp/myimage + imgutil pack /tmp/myimage myprofile + +Build from an image that CI pushed to a registry: + + imgutil build -s docker://registry.example.com/hpc/compute:latest /tmp/myimage + +Build from an Apptainer image: + + imgutil build -s /var/tmp/compute.sif /tmp/myimage + +Prepare an image of the build system OS in CI that builds without network +access: + + imgutil build --containerfile Containerfile + podman build -t localhost/compute . + podman save -o compute.tar localhost/compute + imgutil build -s compute.tar /tmp/compute + +The same for EL10 as a SIF: + + imgutil build --containerfile --apptainer -s alma-10.0-x86_64 compute.def + apptainer build compute.sif compute.def + imgutil build -s compute.sif /tmp/compute + Execute a shell in an unpacked image: imgutil exec /tmp/myimage diff --git a/imgutil/imgutil b/imgutil/imgutil index 2422f30b..869f4bcb 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -22,6 +22,7 @@ import stat import struct import subprocess import sys +import tarfile import tempfile import time try: @@ -654,6 +655,13 @@ class OsHandler(object): selinuxsupport = False # downloaded packages and repository metadata left by the package manager pkgcaches = () + # set by import, the target is an existing root that installs packages + # with its own package manager and repositories + imported = False + # shell snippets for has_packages: list every installed name once, then + # look up each package in it + pkgsetup = 'inst=$(rpm -qa --qf \'[%{PROVIDES}\\n]%{NAME}.%{ARCH}\\n\'); ' + pkgquery = 'printf "%s\\n" "$inst" | grep -xF -- "$p"' def __init__(self, name, version, arch, args): self.name = name @@ -765,6 +773,54 @@ class OsHandler(object): retpkgs.extend(self.list_packages(addpkglist)) return retpkgs + def copy_to_target(self, src, dst): + if self.imported: + # the image may already have it + copy_tree(src, dst) + return + try: + shutil.copytree(src, dst) + except Exception: + sys.stderr.write('Target directory already exists, cannot build\n') + raise + + def has_packages(self, args): + # an imported image that already has every package skips its package + # manager, and with it the need to reach its repositories + args.cmd = ['sh', '-c', self.pkgsetup + 'm=; for p; do ' + self.pkgquery + ' >/dev/null 2>&1 || m="$m $p"; done; ' + '[ -z "$m" ] || { echo "Not in the image, installing with the package manager:$m"; exit 1; }', + 'sh'] + self.packages() + if run_constrainedx(fancy_chroot, (args, self.targpath)): + return False + sys.stdout.write('All packages are already in the image, skipping package installation\n') + sys.stdout.flush() + return True + + def snapshot_initrds(self, initrdname): + # the initramfs of each kernel before the package manager runs + initrds = {} + moddir = os.path.join(self.targpath, 'lib/modules') + for kver in os.listdir(moddir) if os.path.isdir(moddir) else []: + initrd = os.path.join(self.targpath, initrdname.format(kver)) + initrds[kver] = os.path.getmtime(initrd) if os.path.exists(initrd) else None + return initrds + + def stale_kernels(self, initrds, initrdname): + # kernels of an imported image that need their initramfs built: it is + # missing, or unchanged since before the package manager ran and so + # without confluent. One the package manager built or updated has it. + # A modules directory left behind by a removed kernel is skipped. + moddir = os.path.join(self.targpath, 'lib/modules') + for kver in sorted(os.listdir(moddir)) if os.path.isdir(moddir) else []: + bootkernel = os.path.join(self.targpath, 'boot/vmlinuz-' + kver) + if not os.path.exists(bootkernel) and not os.path.exists(os.path.join(moddir, kver, 'vmlinuz')): + continue + initrd = os.path.join(self.targpath, initrdname.format(kver)) + if (os.path.exists(bootkernel) and os.path.exists(initrd) + and os.path.getmtime(initrd) != initrds.get(kver)): + continue + yield kver + class SuseHandler(OsHandler): pkgcaches = ('var/cache/zypp/*',) @@ -785,6 +841,9 @@ class SuseHandler(OsHandler): def add_pkglists(self): self.zyppargs.extend(self.list_packages()) + def packages(self): + return self.zyppargs + def set_source(self, sourcepath): self.sources.append('file://' + sourcepath) enterprise = False @@ -806,7 +865,7 @@ class SuseHandler(OsHandler): def prep_root(self, args): gpgkeys = [] mkdirp(self.targpath) - if not self.sources: + if not self.sources and not self.imported: gpgkeys = glob.glob('/usr/lib/rpm/gnupg/keys/*.asc') targzypp = os.path.join(self.targpath, 'etc/zypp') mkdirp(targzypp) @@ -861,13 +920,20 @@ class SuseHandler(OsHandler): mkdirp(os.path.join(self.targpath, 'etc/dracut.conf.d')) dracutdir = os.path.join(mydir, 'dracut') targdir = os.path.join(self.targpath, 'usr/lib/dracut/modules.d/97diskless') - shutil.copytree(dracutdir, targdir) + self.copy_to_target(dracutdir, targdir) with open(os.path.join(self.targpath, 'etc/dracut.conf.d/diskless.conf'), 'w') as dracutconf: dracutconf.write('compress=xz\nhostonly=no\ndracutmodules+=" diskless base terminfo "\n') cmd = ['chmod', 'a+x'] cmd.extend(glob.glob(os.path.join(targdir, '*'))) subprocess.check_call(cmd) - if self._interactive: + if self.imported: + initrds = self.snapshot_initrds('boot/initrd-{0}') + if not self.has_packages(args): + args.cmd = ['zypper'] if self._interactive else ['zypper', '-n'] + args.cmd += ['install'] + self.zyppargs + if run_constrainedx(fancy_chroot, (args, self.targpath)): + raise Exception('Package installation failed') + elif self._interactive: subprocess.check_call(self.zypper_cmd() + ['install'] + self.zyppargs) else: subprocess.check_call(self.zypper_cmd() + ['-n', 'install'] + self.zyppargs) @@ -881,10 +947,16 @@ class SuseHandler(OsHandler): keysign = 'usr/libexec/ssh/ssh-keysign' if not os.path.exists(os.path.join(self.targpath, keysign)): keysign = 'usr/lib/ssh/ssh-keysign' - with open(os.path.join(self.targpath, 'etc/permissions.local'), 'a') as permout: - permout.write( - '/{0} root:root 4711\n'.format(keysign) - ) + permline = '/{0} root:root 4711'.format(keysign) + # an imported image may have it already + permfile = os.path.join(self.targpath, 'etc/permissions.local') + havelines = [] + if os.path.exists(permfile): + with open(permfile) as permin: + havelines = permin.read().splitlines() + if permline not in havelines: + with open(permfile, 'a') as permout: + permout.write(permline + '\n') if os.path.exists(os.path.join(self.targpath, 'usr/bin/permctl')): args.cmd = ['permctl', '--system', '--set'] else: @@ -893,6 +965,12 @@ class SuseHandler(OsHandler): self.targpath)) + if self.imported: + for kver in self.stale_kernels(initrds, 'boot/initrd-{0}'): + args.cmd = ['dracut', '-f', '/boot/initrd-{0}'.format(kver), kver] + if run_constrainedx(fancy_chroot, (args, self.targpath)): + raise Exception('Building the initramfs of {0} failed'.format(kver)) + return if os.path.exists(os.path.join(self.targpath, 'sbin/mkinitrd')): args.cmd = ['mkinitrd'] else: @@ -941,9 +1019,16 @@ class DebHandler(OsHandler): needapt = 'Missing packages needed in target for capture, to add required packages: apt install ' + ' '.join(needpkgs) self.captureprereqs.append(needapt) + # a name can be provided by a renamed package, libuv1 by libuv1t64 + pkgsetup = ('inst=$(dpkg-query -W -f=\'${db:Status-Status} ${Package} ${Provides}\\n\' | ' + 'awk \'$1 == "installed" { for (i = 2; i <= NF; i++) { sub(/,$/, "", $i); print $i } }\'); ') + def add_pkglists(self): self.includepkgs.extend(self.list_packages()) + def packages(self): + return self.includepkgs + def set_target(self, targpath): self.targpath = targpath @@ -952,12 +1037,10 @@ class DebHandler(OsHandler): mydir = get_mydir(self.oscategory) srcdir = os.path.join(mydir, 'initramfs-tools') targdir = os.path.join(self.targpath, 'etc/initramfs-tools') - try: - shutil.copytree(srcdir, targdir) - except Exception: - sys.stderr.write('Target directory already exists, cannot build\n') - raise + self.copy_to_target(srcdir, targdir) os.chmod(os.path.join(targdir, 'hooks/confluent'), 0o755) + if self.imported: + return #cmd = ['debootstrap', '--include={0}'.format(','.join(self.includepkgs)), self.codename, self.targpath] if self.arch != os.uname().machine: cmd = ['debootstrap', '--arch={0}'.format(deb_arch_map.get(self.arch, self.arch)), self.codename, self.targpath] @@ -975,41 +1058,55 @@ class DebHandler(OsHandler): raise Exception('apt failed in image') def prep_root(self, args): - mkdirp(os.path.join(self.targpath, 'etc/apt')) - for aptdir in ('sources.list.d', 'trusted.gpg.d', 'keyrings'): - mkdirp(os.path.join(self.targpath, 'etc/apt', aptdir)) - shutil.copy('/etc/apt/sources.list', os.path.join(self.targpath, 'etc/apt/sources.list')) - for listfile in glob.glob('/etc/apt/sources.list.d/*'): - shutil.copy(listfile, os.path.join(self.targpath, listfile[1:])) - for keydir in ('/etc/apt/trusted.gpg.d', '/etc/apt/keyrings'): - for keyfile in glob.glob(os.path.join(keydir, '*')): - if not os.path.isfile(keyfile): - continue - tkeyfile = os.path.join( - self.targpath, os.path.relpath(keyfile, '/')) - mkdirp(os.path.dirname(tkeyfile)) - shutil.copy(keyfile, tkeyfile) - if os.path.exists('/etc/apt/trusted.gpg'): - trustedgpg = os.path.join(self.targpath, 'etc/apt/trusted.gpg') - mkdirp(os.path.dirname(trustedgpg)) - shutil.copy( - '/etc/apt/trusted.gpg', trustedgpg) - self._apt(args, ['apt-get', 'update']) - if self.includepkgs: - self._apt(args, ['apt-get', '-y', 'install'] + self.includepkgs) + # an imported image brings its own sources + if not self.imported: + mkdirp(os.path.join(self.targpath, 'etc/apt')) + for aptdir in ('sources.list.d', 'trusted.gpg.d', 'keyrings'): + mkdirp(os.path.join(self.targpath, 'etc/apt', aptdir)) + shutil.copy('/etc/apt/sources.list', os.path.join(self.targpath, 'etc/apt/sources.list')) + for listfile in glob.glob('/etc/apt/sources.list.d/*'): + shutil.copy(listfile, os.path.join(self.targpath, listfile[1:])) + for keydir in ('/etc/apt/trusted.gpg.d', '/etc/apt/keyrings'): + for keyfile in glob.glob(os.path.join(keydir, '*')): + if not os.path.isfile(keyfile): + continue + tkeyfile = os.path.join( + self.targpath, os.path.relpath(keyfile, '/')) + mkdirp(os.path.dirname(tkeyfile)) + shutil.copy(keyfile, tkeyfile) + if os.path.exists('/etc/apt/trusted.gpg'): + trustedgpg = os.path.join(self.targpath, 'etc/apt/trusted.gpg') + mkdirp(os.path.dirname(trustedgpg)) + shutil.copy( + '/etc/apt/trusted.gpg', trustedgpg) + if self.imported: + initrds = self.snapshot_initrds('boot/initrd.img-{0}') + if not (self.imported and self.has_packages(args)): + self._apt(args, ['apt-get', 'update']) + if self.includepkgs: + self._apt(args, ['apt-get', '-y', 'install'] + self.includepkgs) + if self.imported: + for kver in self.stale_kernels(initrds, 'boot/initrd.img-{0}'): + # update mode skips a kernel whose initramfs the image build removed + mode = '-u' if os.path.exists(os.path.join(self.targpath, 'boot/initrd.img-' + kver)) else '-c' + args.cmd = ['update-initramfs', mode, '-k', kver] + if run_constrainedx(fancy_chroot, (args, self.targpath)): + raise Exception('Regenerating the initramfs failed') servicefile = os.path.join( self.targpath, 'usr/lib/systemd/system/ssh.service') if os.path.exists(servicefile): targfile = os.path.join( self.targpath, 'etc/systemd/system/multi-user.target.wants/ssh.service') - if not os.path.exists(targfile): + if not os.path.lexists(targfile): + mkdirp(os.path.dirname(targfile)) os.symlink('/usr/lib/systemd/system/ssh.service', targfile) else: targfile = os.path.join( self.targpath, 'etc/systemd/system/multi-user.target.wants/sshd.service') - if not os.path.exists(targfile): + if not os.path.lexists(targfile): + mkdirp(os.path.dirname(targfile)) os.symlink('/usr/lib/systemd/system/sshd.service', targfile) @@ -1062,6 +1159,12 @@ class ElHandler(OsHandler): # python3 label files written at boot self.yumargs.extend(['selinux-policy-targeted', 'policycoreutils', 'libselinux-utils', 'python3']) + def packages(self): + # set_target puts the options before 'install', the packages follow + if 'install' in self.yumargs: + return self.yumargs[self.yumargs.index('install') + 1:] + return self.yumargs + def set_source(self, sourcepath): yumconfig = create_yumconf(sourcepath, self.addrepos) self.yumargs.extend( @@ -1073,30 +1176,49 @@ class ElHandler(OsHandler): def set_target(self, targpath): self.targpath = targpath - self.yumargs.extend( - ['--installroot={0}'.format(targpath), - '--releasever={0}'.format(self.version)]) - if self.arch != os.uname().machine: - self.yumargs.append('--forcearch={0}'.format(self.arch)) + if not self.imported: + self.yumargs.extend( + ['--installroot={0}'.format(targpath), + '--releasever={0}'.format(self.version)]) + if self.arch != os.uname().machine: + self.yumargs.append('--forcearch={0}'.format(self.arch)) self.yumargs.append('install') def prep_root(self, args): mkdirp(os.path.join(self.targpath, 'usr/lib/dracut/modules.d')) mkdirp(os.path.join(self.targpath, 'etc/dracut.conf.d')) - open(os.path.join(self.targpath, 'etc/resolv.conf'),'w').close() + # an imported image may link it to a path that only resolves in the image + resolv = os.path.join(self.targpath, 'etc/resolv.conf') + if not os.path.lexists(resolv): + open(resolv, 'w').close() mydir = get_mydir(self.oscategory) dracutdir = os.path.join(mydir, 'dracut') targdir = os.path.join(self.targpath, 'usr/lib/dracut/modules.d/97diskless') - shutil.copytree(dracutdir, targdir) + self.copy_to_target(dracutdir, targdir) with open(os.path.join(self.targpath, 'etc/dracut.conf.d/diskless.conf'), 'w') as dracutconf: dracutconf.write('compress=xz\nhostonly=no\ndracutmodules+=" diskless base terminfo "\n') cmd = ['chmod', 'a+x'] cmd.extend(glob.glob(os.path.join(targdir, '*'))) subprocess.check_call(cmd) - if self._interactive: - subprocess.check_call(['yum'] + self.yumargs) + yumcmd = ['yum'] if self._interactive else ['yum', '-y'] + if self.imported: + initrds = self.snapshot_initrds('boot/initramfs-{0}.img') + if not self.has_packages(args): + args.cmd = yumcmd + self.yumargs + if run_constrainedx(fancy_chroot, (args, self.targpath)): + raise Exception('Package installation failed') + # a container build may also not have put the kernel in /boot + for kver in self.stale_kernels(initrds, 'boot/initramfs-{0}.img'): + modkernel = 'lib/modules/{0}/vmlinuz'.format(kver) + if os.path.exists(os.path.join(self.targpath, modkernel)): + args.cmd = ['kernel-install', 'add', kver, '/' + modkernel] + else: + # only /boot has the kernel, so only the initramfs is missing + args.cmd = ['dracut', '-f', '/boot/initramfs-{0}.img'.format(kver), kver] + if run_constrainedx(fancy_chroot, (args, self.targpath)): + raise Exception('Installing kernel {0} failed'.format(kver)) else: - subprocess.check_call(['yum', '-y'] + self.yumargs) + subprocess.check_call(yumcmd + self.yumargs) # boot entries written in the chroot carry the kernel command line of # the build system, and installing to disk writes its own entries for loadent in glob.glob(os.path.join(self.targpath, 'boot/loader/entries/*.conf')): @@ -1365,7 +1487,8 @@ def main(): buildp.add_argument('-p', '--packagelist', help='Filename of package list to replace default pkglist', default='') buildp.add_argument('-a', '--addpackagelist', action='append', default=[], help='A list of additional packages to include, may be specified multiple times') - buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used. For Ubuntu, this is not supported, the build system repositories are always used.') + buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used. For Ubuntu, a distribution directory is not supported, the build system repositories are always used. ' + 'A container image made elsewhere, for example in CI, can be the source instead: a SIF file, an OCI or Docker archive, an OCI layout directory, or a podman reference with its transport such as docker://registry.example.com/compute:latest or containers-storage:localhost/compute') buildp.add_argument('--arch', default=None, help='Target architecture to build for (e.g. aarch64). When it differs from the build host architecture, statically linked qemu-user emulator registered in binfmt_misc with the F flag is required. aarch64 is emulated as a Neoverse N1 cpu (qemu 7.1 or later), set QEMU_CPU to choose another') buildp.add_argument('-y', '--non-interactive', help='Avoid prompting for confirmation', action='store_true') buildp.add_argument('--selinux', action='store_true', @@ -1375,7 +1498,14 @@ def main(): help='Directory to make available in the build environment. -v / will ' 'cause it to be mounted in image as /run/external/, -v /:/run/root ' 'will override the target to be /run/root, and something like /var/lib/repository:- will cause it to mount to the identical path inside the image', action='append') - buildp.add_argument('scratchdir', help='Directory to build new diskless root in') + buildp.add_argument('--containerfile', action='store_true', + help='Write a Containerfile instead of building, for an image with every package build checks for, ' + 'so that building from the image needs no repositories') + buildp.add_argument('--apptainer', action='store_true', + help='With --containerfile, write an Apptainer definition file for building a SIF instead') + buildp.add_argument('--from', dest='base', default=None, + help='With --containerfile, the base image to use instead of the default for the OS') + buildp.add_argument('scratchdir', help='Directory to build new diskless root in, or with --containerfile the file to write, - for standard output') execp = sps.add_parser('exec', help='Start specified scratch directory as container') execp.add_argument('-v', '--volume', help='Directory to make available in the build environment. -v / will ' @@ -1395,13 +1525,24 @@ def main(): capturep.add_argument('node', help='Node to capture image from') capturep.add_argument('profilename', help='Profile name for captured image') args = parser.parse_args() - if hasattr(args, 'scratchdir') and args.scratchdir: + if hasattr(args, 'scratchdir') and args.scratchdir and not ( + getattr(args, 'containerfile', False) and args.scratchdir == '-'): args.scratchdir = os.path.abspath(args.scratchdir) if not args or not args.subcommand: parser.print_usage() sys.exit(1) if args.subcommand == 'build': - build_root(args) + if (args.apptainer or args.base) and not args.containerfile: + sys.stderr.write('--apptainer and --from require --containerfile\n') + sys.exit(1) + if args.containerfile and args.addrepos: + sys.stderr.write('-r is not supported with --containerfile, the image installs ' + 'from the repositories of its base image\n') + sys.exit(1) + if args.source and container_source(args.source): + import_root(args) + else: + build_root(args) elif args.subcommand == 'capture': capture_remote(args) elif args.subcommand == 'unpack': @@ -1557,6 +1698,10 @@ qemu_arch_map = { 'aarch64': 'qemu-aarch64', 'x86_64': 'qemu-x86_64', } +elf_arch_map = { + 62: 'x86_64', + 183: 'aarch64', +} def set_qemu_cpu(targetarch): @@ -1806,19 +1951,20 @@ def fingerprint_host(args, hostpath='/'): return oshandler return oshandler -def build_root(args): - if args.scratchdir: - args.scratchdir = os.path.abspath(args.scratchdir) - check_root(args.scratchdir) - yumargs = ['yum', '--installroot={0}'.format(args.scratchdir)] - if args.source: - if '/' not in args.source and not os.path.exists(args.source): - args.source = os.path.join('/var/lib/confluent/distributions/', args.source) - oshandler = fingerprint_source(args.source, args) - if oshandler is not None: - oshandler.set_source(args.source) - else: - oshandler = fingerprint_host(args) +def build_root(args, oshandler=None): + if not args.containerfile: + check_root(args.scratchdir) + # an imported image comes with its handler + if oshandler is None: + if args.source: + if '/' not in args.source and not os.path.exists(args.source): + args.source = os.path.join('/var/lib/confluent/distributions/', args.source) + oshandler = fingerprint_source(args.source, args) + # a Containerfile installs from the repositories of its base image + if oshandler is not None and not args.containerfile: + oshandler.set_source(args.source) + else: + oshandler = fingerprint_host(args) if oshandler is None: if args.source: sys.stderr.write( @@ -1829,13 +1975,16 @@ def build_root(args): 'Unable to recognize build system os\n') sys.exit(1) hostarch = os.uname().machine - if args.arch and args.arch != oshandler.arch: + if args.arch and uname_arch_map.get(args.arch, args.arch) != oshandler.arch: if args.source: sys.stderr.write( - 'Requested architecture {0} conflicts with source directory ' + 'Requested architecture {0} conflicts with source ' 'architecture {1}\n'.format(args.arch, oshandler.arch)) sys.exit(1) oshandler.set_arch(args.arch) + if args.containerfile: + write_containerfile(args, oshandler) + return if oshandler.arch != hostarch: if not oshandler.crossbuild: sys.stderr.write( @@ -1867,6 +2016,290 @@ def build_root(args): #if len(args) > 1: # pack_image(opts, args) +container_minversions = {'el': 9, 'ubuntu': 24.04, 'suse': 16} +# podman transports for a container image that is not a local file +container_transports = ('containers-storage:', 'dir:', 'docker://', 'docker-archive:', + 'oci:', 'oci-archive:') + +def container_supported(oscategory): + """Whether building from container images and generating a Containerfile + support oscategory, for example el10 or ubuntu24.04""" + match = re.match(r'^(el|ubuntu|suse)([0-9]+(\.[0-9]+)?)$', oscategory) + return bool(match) and float(match.group(2)) >= container_minversions[match.group(1)] + +def container_source(source): + """Whether the build source is a container image rather than a + distribution directory""" + if os.path.isdir(source): + return os.path.exists(os.path.join(source, 'oci-layout')) + return os.path.isfile(source) or source.startswith(container_transports) + +def get_sif_offset(sifname): + """Return the offset of the squashfs root partition of the SIF file + sifname, or None if it is not a SIF file""" + with open(sifname, 'rb') as sifin: + header = sifin.read(128) + if header[32:41] != b'SIF_MAGIC': + return None + # descriptor count, offset and total size, little endian like the + # rest of the format + dtotal, doffset, dsize = struct.unpack(' os.fstat(sifin.fileno()).st_size: + raise Exception('{0} is not a valid SIF file'.format(sifname)) + for idx in range(dtotal): + sifin.seek(doffset + idx * (dsize // dtotal)) + # data type, in use flag, three ids, then the data offset + dtype, used, offset = struct.unpack('