diff --git a/.gitignore b/.gitignore index 14ef952..d33852c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.log *-ppa build-logs* +.*~ diff --git a/do_autopkg_tests.sh b/do_autopkg_tests.sh index c9b0de3..9f25c1d 100755 --- a/do_autopkg_tests.sh +++ b/do_autopkg_tests.sh @@ -5,17 +5,18 @@ declare -A all_series all_series=( - ['bionic']="ubuntu:bionic" +# ['bionic']="ubuntu:bionic" # ['focal']="ubuntu:focal" # ['jammy']="ubuntu:jammy" # ['noble']="ubuntu:noble" -# ['oracular']="ubuntu:oracular" -# ['plucky']="ubuntu-daily:devel" +# ['plucky']="ubuntu:plucky" + ['questing']="ubuntu:questing" +# ['resolute']="ubuntu-daily:devel" ) -latest_dev_series="plucky" +latest_dev_series="resolute" latest_sup_series="noble" -non_lts_releases="oracular" +non_lts_releases="questing" ppa_user="arif-ali" ppa_name="sosreport" @@ -43,7 +44,7 @@ do image_list="$(lxc image list --format json | jq -rc '.[] | select(.aliases[].name | contains("autopkgtest")) | .aliases[].name' | xargs)" - if [[ $(echo ${image_list} | grep -v "autopkgtest/ubuntu/${distro}/amd64/vm") ]] ; then + if [[ '$(echo ${image_list} | grep -v "autopkgtest/ubuntu/${distro}/amd64/vm")' ]] ; then autopkgtest-build-lxd --vm ${image} fi @@ -56,7 +57,7 @@ do --no-built-binaries --ignore-restrictions=isolation-machine \ --setup-commands="sudo apt -y install software-properties-common ; \ sudo add-apt-repository -y -u -s ppa:${ppa_user}/${ppa_name}" \ - sosreport -- ${autpkgtest_cmd} + sos -- ${autpkgtest_cmd} done diff --git a/do_sos_avocado.sh b/do_sos_avocado.sh index 2a85eba..b28ed1b 100755 --- a/do_sos_avocado.sh +++ b/do_sos_avocado.sh @@ -4,6 +4,8 @@ script_cmd=${0##*/} +. functions.sh + declare -A all_series if [[ "${script_cmd}" == *"_rh.sh" ]]; then @@ -27,29 +29,30 @@ else # ['bionic']="ubuntu:bionic" # ['focal']="ubuntu:focal" # ['jammy']="ubuntu:jammy" - ['noble']="ubuntu:noble" - # ['oracular']="ubuntu:oracular" - # ['plucky']="ubuntu-daily:devel" + # ['noble']="ubuntu:noble" + # ['plucky']="ubuntu:plucky" + ['questing']="ubuntu:questing" + # ['resolute']="ubuntu-daily:resolute" ) project_name="sos-testing-ubuntu" lxd_profile="lxd_profile_ubuntu.yaml" image_pre="" - latest_dev_series="plucky" + latest_dev_series="questing" latest_sup_series="noble" - non_lts_releases="oracular" - - sos_type="deb" + non_lts_releases="oracular plucky" snap_channel="latest/candidate" ppa_user="arif-ali" ppa_name="sosreport-dev" + # this can be "deb" or "snap" + sos_type="deb" # this can be "ppa" or "proposed" - deb_type="ppa" + deb_type="proposed" -fi +fi # general variables @@ -83,7 +86,7 @@ do cd - fi - check-vm-status ${inst_name} ${project_name} + check_vm_status ${inst_name} ${project_name} if [[ "${script_cmd}" == "do_sos_avocado"* ]] ; then ${lxc} file push /tmp/sos.tgz ${inst_name}/root/sos.tgz @@ -115,9 +118,15 @@ do #!/bin/bash sos_type="${sos_type}" +deb_type="${deb_type}" +distro="${distro}" +proposed="" if [[ "\${sos_type}" == "deb" ]] ; then - apt -y install sosreport + if [[ "\${deb_type}" == "proposed" ]] ; then + proposed="/\${distro}-proposed" + fi + apt -y install sosreport\${proposed} elif [[ "\${sos_type}" == "snap" ]] ; then snap install sosreport --channel ${snap_channel} --classic elif [[ "\${sos_type}" == "dnf" ]] ; then @@ -127,6 +136,12 @@ EOF ${lxc} file push setup_sos.sh ${inst_name}/root/setup_sos.sh fi +sleep_time=120 + +# Waiting 60 seconds for some logs to be created +echo Waiting ${sleep_time} for logs and VM to settle ... +sleep ${sleep_time} + if [[ "${script_cmd}" == "do_sos_avocado"* ]] ; then ${lxc_exec} ${inst_name} -- bash /root/run_avocado.sh | tee sos_run_avocado_${distro}.log elif [[ "${script_cmd}" == "do_sos_testing"* ]] ; then diff --git a/do_sos_deb.sh b/do_sos_deb.sh index f992eab..1f676fc 100755 --- a/do_sos_deb.sh +++ b/do_sos_deb.sh @@ -7,7 +7,8 @@ script_cmd=${0##*/} declare -A all_series all_series=( - ['sid']="debian/sid/cloud" + ['debian']="images:debian/sid/cloud" + ['ubuntu']="ubuntu:resolute" ) project_name="deb-testing" @@ -15,11 +16,16 @@ project="--project ${project_name}" lxc="lxc ${project}" lxc_exec="${lxc} exec" fresh="false" +distro="debian" show_help() { echo " -p The product need testing + -o The orig file to download + -d The distro to run on + One of: ubuntu, debian + Default: debian -f Create a fresh instance " } @@ -31,7 +37,7 @@ if [ $# -eq 0 ]; then exit 0 fi -while getopts ":p:f" opt; do +while getopts ":p:o:d:f" opt; do case $opt in p) prod="$OPTARG" @@ -39,6 +45,12 @@ while getopts ":p:f" opt; do f) fresh="true" ;; + o) + origin="$OPTARG" + ;; + d) + distro="$OPTARG" + ;; :) printf "Option -%s needs an argument.\n" "$OPTARG" >&2 show_help @@ -48,22 +60,22 @@ while getopts ":p:f" opt; do esac done -prod_location="$HOME/packaging/${prod}" +if [[ $distro == "debian" ]] ; then + prod_location="$HOME/packaging/${prod}" +else + prod_location="$HOME/packaging/git-ubuntu/${prod}" +fi lxc project create ${project_name} ${lxc} profile edit default < lxd_profile_deb.yaml -for distro in "${!all_series[@]}" -do - inst_name=${prod}-${distro} - inst_name=autopkgtest if [[ ${fresh} == "true" ]] ; then - image="images:${all_series[${distro}]}" + image="${all_series[${distro}]}" ${lxc} delete ${inst_name} --force - ${lxc} launch ${image} ${inst_name} --vm -c limits.cpu=4 -c limits.memory=4GiB -s virtual + ${lxc} launch ${image} ${inst_name} --vm -c limits.cpu=4 -c limits.memory=4GiB -d root,size=50GiB else ${lxc} start ${inst_name} fi @@ -106,12 +118,12 @@ do ${lxc_exec} ${inst_name} -- sudo apt -y upgrade fi - ${lxc_exec} ${inst_name} -- bash /root/scripts/deb_build_check.sh ${prod} + ${lxc_exec} ${inst_name} -- usermod -G sbuild root + ${lxc_exec} ${inst_name} -- id root + ${lxc_exec} ${inst_name} -- bash /root/scripts/deb_build_check.sh ${prod} ${origin} ${lxc} file pull ${inst_name}/root/logs/build-logs-${prod}.tgz build-logs-${distro}-${prod}.tgz ${lxc} stop ${inst_name} rm -rf build-logs-${distro}-${prod} mkdir -p build-logs-${distro}-${prod} tar -C build-logs-${distro}-${prod} -xf build-logs-${distro}-${prod}.tgz - -done diff --git a/get_image_names.sh b/get_image_names.sh index 99675ff..57ed785 100755 --- a/get_image_names.sh +++ b/get_image_names.sh @@ -27,13 +27,19 @@ project="ubuntu-os-cloud" check_distro "$families" "$project" -latest_sup_rev="2504" +# Check Debian stable images +families="debian-11 debian-12 debian-13" +project="debian-cloud" + +check_distro "$families" "$project" + +latest_sup_rev="2510" gcloud compute images list --project ubuntu-os-cloud --filter="family:ubuntu-${latest_sup_rev}-amd64" gcloud compute images list --project ubuntu-os-cloud --filter="family:ubuntu-minimal-${latest_sup_rev}-amd64" # For testing for the next daily release depending on LTS or non-LTS -dev_rel=2510 +dev_rel=2604-lts lts_rel=2404 gcloud compute images list --project ubuntu-os-cloud-devel --filter="family:ubuntu-${dev_rel}-amd64" diff --git a/lxd_profile_deb.yaml b/lxd_profile_deb.yaml index 2cc02d5..af062d0 100644 --- a/lxd_profile_deb.yaml +++ b/lxd_profile_deb.yaml @@ -5,11 +5,13 @@ devices: type: nic root: path: / - pool: virtual + pool: default type: disk config: user.user-data: | #cloud-config + groups: + - sbuild: [root] packages: - python3-pip - tox @@ -18,6 +20,7 @@ config: - devscripts - piuparts - equivs + - sbuild write_files: - path: /root/scripts/setup.sh permissions: '0755' @@ -33,6 +36,9 @@ config: #$run_autopkgtest = 1; $autopkgtest_root_args = ''; $autopkgtest_opts = [ '--apt-upgrade', '--', 'unshare', '--release', '%r', '--arch', '%a' ]; + + # don't remove this, Perl needs it: + 1; - path: /root/data/.sbuildrc_sbuild owner: root:root content: | @@ -41,6 +47,11 @@ config: $build_source = 1; + # Ensure sbuild uses /var/tmp for its temporary files + $build_environment = { + 'TMPDIR' => '/var/tmp', + }; + # --source-only-changes (applicable for dput. irrelevant for dgit push-source). $source_only_changes = 1; @@ -63,11 +74,11 @@ config: $lintian_opts = ['-v', '-i', '-I', '-E', '--pedantic', '--profile=debian']; $run_piuparts = 1; - $piuparts_opts = ['--schroot', '%r-%a-sbuild', '--no-eatmydata']; + $piuparts_opts = ['--schroot', '%r-%a', '--no-eatmydata']; $run_autopkgtest = 1; $autopkgtest_root_args = ''; - $autopkgtest_opts = [ '--', 'schroot', '%r-%a-sbuild' ]; + $autopkgtest_opts = [ '--', 'schroot', '%r-%a' ]; # When to purge the build directory afterwards; possible values are "never", # "successful", and "always". "always" is the default. It can be helpful @@ -94,6 +105,11 @@ config: $build_source = 1; + # Ensure sbuild uses /var/tmp for its temporary files + $build_environment = { + 'TMPDIR' => '/var/tmp', + }; + # --source-only-changes (applicable for dput. irrelevant for dgit push-source). $source_only_changes = 1; @@ -143,19 +159,14 @@ config: content: | #!bin/bash - LICENSECHECK=1 - PBUILDER=1 - LINTIAN=1 - REPROTEST=1 - SBUILD=1 - SBUILD_UNSHARE=1 - prod="${1}" + origin="${2}" build_dir="/root/build" prod_build_dir="${build_dir}/${prod}" log_dir="/root/logs" + profile="ubuntu" - rm -rf ~/.cache/sbuild + rm -rf ~/.cache/sbuild ${prod_build_dir} ${log_dir} mkdir -p ~/.cache/sbuild ${prod_build_dir} ${log_dir} cd ${prod_build_dir} @@ -163,6 +174,20 @@ config: series=$(head -n 1 debian/changelog | awk '{print $3}' | sed 's/;//g' | awk -F'-' '{print $1}') + LICENSECHECK=0 + PBUILDER=0 + LINTIAN=0 + REPROTEST=0 + if [[ "${series}" == "unstable" ]] ; then + LICENSECHECK=1 + PBUILDER=1 + LINTIAN=1 + REPROTEST=1 + profile="debian" + fi + SBUILD=1 + SBUILD_UNSHARE=1 + if [[ $LICENSECHECK -eq 1 ]] ; then apt -y install licenserecon lrc | tee ${log_dir}/license-check.log @@ -171,25 +196,34 @@ config: export EMAIL="Some One " export DEBEMAIL="Some One " - uscan --download-current-version --rename - dpkg-buildpackage -S -d - mk-build-deps - apt -y install ./*-build-deps*.deb + if [[ -n "${origin}" ]] ; then + cd ../ + wget ${origin} -O ${origin##*/} + cd - + else + uscan --download-current-version --rename + fi + apt -y build-dep . + dpkg-buildpackage -S dpkg-buildpackage -b -us -uc -rfakeroot -m - rm -rf ./*-build-deps* if [[ $LINTIAN -eq 1 ]] ; then apt -y install lintian cd ${prod_build_dir} - lintian -v -i -I -E --pedantic --profile debian ../*.dsc \ + lintian -v -i -I -E --pedantic --profile ${profile} ../*.dsc \ | tee ${log_dir}/lintian-dsc.log - lintian -v -i -I -E --pedantic --profile debian ../*.changes \ + lintian -v -i -I -E --pedantic --profile ${profile} ../*.changes \ | tee ${log_dir}/lintian-changes.log - lintian -v -i -I -E --pedantic --profile debian ../*.buildinfo \ + lintian -v -i -I -E --pedantic --profile ${profile} ../*.buildinfo \ | tee ${log_dir}/lintian-buildinfo.log cd - fi + rm -rf ${prod_build_dir} + mkdir -p ${prod_build_dir} + cd ${prod_build_dir} + tar xfz ../${prod}.tgz + if [[ $PBUILDER -eq 1 ]] ; then apt install -y pbuilder cd ${prod_build_dir} @@ -197,13 +231,18 @@ config: pbuilder build --distribution ${series} --twice ../*.dsc \ | tee ${log_dir}/pbuilder.log cd - + pbuilder --clean fi if [[ $REPROTEST -eq 1 ]] ; then apt install -y reprotest cd ${prod_build_dir} - chroot="${series}-amd64-sbuild" + chroot="${series}-amd64" sbuild-destroychroot ${chroot} + rm --recursive --one-file-system /srv/chroot/${chroot} + rm /etc/sbuild/chroot/${chroot} + rm /etc/schroot/chroot.d/${chroot}* + sbuild-createchroot --include=eatmydata,ccache ${series} /srv/chroot/${chroot} reprotest --print-sudoers \ @@ -220,25 +259,39 @@ config: fi if [[ $SBUILD -eq 1 ]] ; then - apt install -y sbuild mmdebstrap uidmap - chroot="${series}-amd64-sbuild" - sbuild-destroychroot ${chroot} + apt install -y sbuild mmdebstrap uidmap autopkgtest + [[ "${series}" != "sid" ]] && apt -y install ubuntu-dev-tools + chroot="${series}-amd64" cd ${prod_build_dir} cp /root/data/.sbuildrc_sbuild /root/.sbuildrc + sed -i "s/debian/${profile}/g" /root/.sbuildrc - mkdir -p /srv/chroot/${chroot} - sbuild-createchroot --include=eatmydata,ccache ${series} /srv/chroot/${chroot} + + if [[ "${series}" == "sid" ]] ; then + sbuild-destroychroot ${chroot} + mkdir -p /srv/chroot/${chroot} + sbuild-createchroot --include=eatmydata,ccache ${series} /srv/chroot/${chroot} + else + sg sbuild -c "mk-sbuild ${series}" + fi sbuild -d ${series} | tee ${log_dir}/sbuild.log cd - fi if [[ $SBUILD_UNSHARE -eq 1 ]] ; then - apt install -y sbuild mmdebstrap uidmap + apt install -y sbuild mmdebstrap uidmap autopkgtest + [[ "${series}" != "sid" ]] && apt -y install ubuntu-dev-tools cd ${prod_build_dir} cp /root/data/.sbuildrc_unshare /root/.sbuildrc + sed -i "s/debian/${profile}/g" /root/.sbuildrc + + if [[ "${series}" == "sid" ]] ; then + mmdebstrap --variant=buildd ${series} ~/.cache/sbuild/${series}-amd64.tar.zst + else + sg sbuild -c "mk-sbuild ${series}" + fi - mmdebstrap --variant=buildd ${series} ~/.cache/sbuild/${series}-amd64.tar.zst echo "$(id -un):120000:65536" | sudo tee -a /etc/subuid /etc/subgid sbuild -d ${series} | tee ${log_dir}/sbuild_unshare.log diff --git a/lxd_profile_ubuntu.yaml b/lxd_profile_ubuntu.yaml index 9de9856..60ef24b 100644 --- a/lxd_profile_ubuntu.yaml +++ b/lxd_profile_ubuntu.yaml @@ -85,7 +85,7 @@ config: tar xf /tmp/sos*.tar.xz echo "Check any file size that is 0 ..." - find sosreport*/ -type f -size 0 + find sosreport*/ -type f -size 0 | grep -v -E '/sys/|/proc/' echo "Check any non INFO messages in sos.log ..." grep -v "INFO:" sosreport*/sos_logs/sos.log diff --git a/ppa_tests.sh b/ppa_tests.sh index 707a3f1..703f246 100755 --- a/ppa_tests.sh +++ b/ppa_tests.sh @@ -4,8 +4,8 @@ PPA="arif-ali/sosreport-dev" # Possible values, amd64,arm64,armhf,ppc64el,s390x archs="amd64,arm64,armhf,ppc64el,s390x" # possible values focal,jammy,noble,oracular,plucky -releases="focal,jammy,noble,oracular,plucky" -package="sosreport" +releases="jammy,noble,questing,resolute" +package="sos,sosreport" ppa tests --dry-run \ --release ${releases} \