Merge pull request #7778 from VersatusHPC/fix/rpm-preserve-xcat-conf

fix(rpm): preserve local xcat.conf changes across upgrades

docs/source/advanced/security/apache_hardening.rst

@@ -107,6 +107,30 @@ provisioning scripts crawl recursively.
    download all files from these directories and depend on Apache directory
    listings to discover file paths.
 
+Local Customizations
+--------------------
+
+``xcat.conf`` belongs to the xCAT package. Keep site-specific rules in a
+separate file that Apache reads after it:
+
+* EL: ``/etc/httpd/conf.d/zz-xcat-local.conf``
+* SUSE: ``/etc/apache2/conf.d/zz-xcat-local.conf``
+
+Apache reads ``conf.d`` in alphabetical order, so the ``zz-`` prefix puts the
+file after ``xcat.conf`` and its directives take precedence. Use the relative
+form of ``Options`` (``+`` or ``-``): it adjusts the options inherited from the
+xCAT block, where the absolute form replaces all of them. For example, to allow
+browsing of one site directory::
+
+    # /etc/httpd/conf.d/zz-xcat-local.conf
+    <Directory "/install/custom/mypkgs">
+        Options +Indexes
+    </Directory>
+
+Editing ``xcat.conf`` in place still works: an upgrade keeps the modified file.
+If the packaged configuration changed too, the new version is written alongside
+it as ``xcat.conf.rpmnew`` and has to be merged by hand.
+
 Security Response Headers and Server Banner
 -------------------------------------------
 

xCAT-test/unit/xcat_conf_upgrade_preservation.t

@@ -0,0 +1,168 @@
+#!/usr/bin/env perl
+# xcat.conf used to be ordinary payload rewritten from a conf.orig template in
+# %post, so an upgrade discarded local edits with no .rpmnew or .rpmsave. Pin
+# the replacement: build-time source selection, config(noreplace) ownership,
+# and a %pretrans migration that only removes a still-unmodified active file.
+# The migration has to be in %pretrans: rpm fixes each config file's fate
+# before %pre, so removing the file there can leave only a .rpmnew behind.
+use strict;
+use warnings;
+
+use File::Spec;
+use FindBin;
+use Test::More;
+
+my $repo_root = File::Spec->catdir( $FindBin::Bin, '..', '..' );
+
+sub read_file {
+    my ($relative_path) = @_;
+    my $path = File::Spec->catfile( $repo_root, split( '/', $relative_path ) );
+    open( my $fh, '<', $path ) or die "Unable to read $path: $!";
+    my $contents = do { local $/; <$fh> };
+    close($fh);
+    return $contents;
+}
+
+# Only real section keywords end a section, so nested %if/%ifos/%define stay put.
+my %SECTION = map { $_ => 1 }
+    qw(description prep build pretrans pre install post preun postun posttrans clean files changelog);
+
+sub section {
+    my ( $spec, $wanted ) = @_;
+    my ( $inside, @body ) = (0);
+    for my $line ( split( /\n/, $spec, -1 ) ) {
+        if ( $line =~ /^%(\w+)/ && $SECTION{$1} ) {
+            $inside = ( $1 eq $wanted );
+            next;
+        }
+        push @body, $line if $inside;
+    }
+    return join( "\n", @body ) . "\n";
+}
+
+my @packages = (
+    {
+        name => 'management-node',
+        spec => 'xCAT/xCAT.spec',
+        # %httpconfigdir redirects these to /etc/xcathttpdsave on s390x.
+        templates => '/etc/%httpconfigdir/conf.orig',
+        apache24  => 7,
+        order     => [ 'httpd', 'apache2' ],
+    },
+    {
+        name      => 'service-node',
+        spec      => 'xCATsn/xCATsn.spec',
+        templates => '/etc/xcat/conf.orig',
+        apache24  => 6,
+        order     => [ 'apache2', 'httpd' ],
+    },
+);
+
+for my $pkg (@packages) {
+    my $label     = $pkg->{name};
+    my $spec      = read_file( $pkg->{spec} );
+    my $templates = quotemeta $pkg->{templates};
+    my $apache24  = $pkg->{apache24};
+    my ( $first, $second ) = @{ $pkg->{order} };
+
+    # The %install selection is by source number, so pin what each one holds.
+    like( $spec, qr{^Source1:\s+xcat\.conf$}m,
+        "$label package still takes the Apache 2.2 configuration from Source1" );
+    like( $spec, qr{^Source$apache24:\s+xcat\.conf\.apach24$}m,
+        "$label package still takes the Apache 2.4 configuration from Source$apache24" );
+
+    my $install = section( $spec, 'install' );
+    like(
+        $install,
+        qr{
+            ^%if \s+ 0%\{\?fedora\} \s* \|\| \s*
+                     0%\{\?rhel\} \s* >= \s* 7 \s* \|\| \s*
+                     0%\{\?suse_version\} \s* >= \s* 1200 \s* $ \n
+            ^cp \s+ %\{SOURCE$apache24\} \s+ \$RPM_BUILD_ROOT/etc/$first/conf\.d/xcat\.conf \s* $ \n
+            ^cp \s+ %\{SOURCE$apache24\} \s+ \$RPM_BUILD_ROOT/etc/$second/conf\.d/xcat\.conf \s* $ \n
+            ^%else \s* $ \n
+            ^cp \s+ %\{SOURCE1\} \s+ \$RPM_BUILD_ROOT/etc/$first/conf\.d/xcat\.conf \s* $ \n
+            ^cp \s+ %\{SOURCE1\} \s+ \$RPM_BUILD_ROOT/etc/$second/conf\.d/xcat\.conf \s* $ \n
+            ^%endif \s* $
+        }mx,
+        "$label package selects the Apache generation at build time, Apache 2.4 on Fedora, EL7+ and SLES 12+"
+    );
+
+    # The next upgrade's %pretrans compares against these, so they must stay shipped.
+    like( $install, qr{^cp \s+ %\{SOURCE$apache24\} \s+ \$RPM_BUILD_ROOT$templates/xcat\.conf\.apach24 \s* $}mx,
+        "$label package still saves the Apache 2.4 template under conf.orig" );
+    like( $install, qr{^cp \s+ %\{SOURCE1\} \s+ \$RPM_BUILD_ROOT$templates/xcat\.conf\.apach22 \s* $}mx,
+        "$label package still saves the Apache 2.2 template under conf.orig" );
+
+    my $files = section( $spec, 'files' );
+    for my $dir ( 'httpd', 'apache2' ) {
+        like( $files, qr{^%config\(noreplace\)\s+/etc/$dir/conf\.d/xcat\.conf\s*$}m,
+            "$label package owns /etc/$dir/conf.d/xcat.conf as config(noreplace)" );
+    }
+    like( $files, qr{^$templates/xcat\.conf\.apach24\s*$}m,
+        "$label package ships the Apache 2.4 template" );
+    like( $files, qr{^$templates/xcat\.conf\.apach22\s*$}m,
+        "$label package ships the Apache 2.2 template" );
+
+    my $post = section( $spec, 'post' );
+    unlike( $post, qr{\brm\b[^\n]*conf\.d/xcat\.conf},
+        "$label %post never removes an active xcat.conf" );
+    unlike( $post, qr{\bcp\b[^\n]*xcat\.conf},
+        "$label %post never copies a template over an active xcat.conf" );
+    unlike( $post, qr{conf\.orig},
+        "$label %post no longer reads the saved templates at all" );
+
+    # The only mention left in %post is the read-only guard around a2enmod.
+    my @post_refs = ( $post =~ m{^([^\n]*conf\.d/xcat\.conf[^\n]*)$}mg );
+    is( scalar @post_refs, 1,
+        "$label %post refers to an active xcat.conf exactly once" );
+    like( $post_refs[0] || '',
+        qr{^\s*if \[ -e /etc/apache2/conf\.d/xcat\.conf \] && command -v a2enmod\b},
+        "$label %post only tests for the file, to enable mod_headers" );
+
+    # rpm decides each config file's fate before %pre, so the migration has to
+    # run in %pretrans or it can delete a file rpm already resolved to .rpmnew.
+    like( $spec, qr{^%pretrans -p <lua>$}m,
+        "$label package migrates in %pretrans, as an embedded Lua scriptlet" );
+    my $pre = section( $spec, 'pre' );
+    unlike( $pre, qr{conf\.d/xcat\.conf},
+        "$label %pre does not touch an active xcat.conf" );
+
+    my $pretrans = section( $spec, 'pretrans' );
+    like( $pretrans, qr{^if arg\[2\] and tonumber\(arg\[2\]\) == 1 then return end$}m,
+        "$label migration returns early on a fresh install" );
+    like(
+        $pretrans,
+        qr{
+            ^local \s+ templates \s* = \s* \{ \s*
+                contents\("$templates/xcat\.conf\.apach24"\), \s* $ \n
+            ^\s* contents\("$templates/xcat\.conf\.apach22"\) \s* \} \s* $
+        }mx,
+        "$label migration reads both of the outgoing package's saved templates"
+    );
+    like(
+        $pretrans,
+        qr{
+            ^for \s+ _, \s* active \s+ in \s+ ipairs\(\{ \s* "/etc/httpd/conf\.d/xcat\.conf", \s* $ \n
+            ^\s* "/etc/apache2/conf\.d/xcat\.conf" \s* \}\) \s+ do \s* $ \n
+            (?: ^\s* --[^\n]* \n )*                    # explanatory comment
+            ^\s* if \s+ posix\.stat\(active, \s* "type"\) \s* == \s* "regular" \s+ then \s* $ \n
+            ^\s* local \s+ current \s* = \s* contents\(active\) \s* $ \n
+            ^\s* for \s+ _, \s* template \s+ in \s+ ipairs\(templates\) \s+ do \s* $ \n
+            ^\s* if \s+ current \s+ and \s+ current \s* == \s* template \s+ then \s* $ \n
+            ^\s* os\.remove\(active\) \s* $
+        }mx,
+        "$label migration removes an active xcat.conf only when it is a regular file whose contents still match a saved template"
+    );
+
+    # Nothing else may delete the active file, in any scriptlet.
+    unlike( $spec, qr{\brm\b[^\n]*conf\.d/xcat\.conf},
+        "$label package never shells out to rm for an active xcat.conf" );
+    my @removals = ( $pretrans =~ m{^\s*(os\.remove\([^\n]*)$}mg );
+    is_deeply( \@removals, ['os.remove(active)'],
+        "$label migration removes exactly one validated path and nothing else" );
+    unlike( $spec, qr{^Requires\(pre\):\s+/usr/bin/cmp$}m,
+        "$label package needs no external tool for the comparison" );
+}
+
+done_testing();

xCAT/xCAT.spec

@@ -125,6 +125,44 @@ tar -xf postscripts.tar
 
 %build
 
+%ifos linux
+# rpm fixes each config file's fate before %pre runs, so a migration there can
+# delete an active file rpm has already decided to leave as xcat.conf.rpmnew.
+# %pretrans runs before that decision.  It is Lua because a pre-transaction
+# scriptlet cannot rely on any dependency being unpacked yet.
+%pretrans -p <lua>
+-- Older packages shipped xcat.conf as ordinary payload and then rewrote it
+-- from conf.orig in %post, so rpm's recorded digest cannot tell a stock file
+-- from a customised one.  Compare against the outgoing package's templates,
+-- still on disk at this point, and drop only a file that still matches one.
+if arg[2] and tonumber(arg[2]) == 1 then return end
+
+local function contents(path)
+    local handle = io.open(path, "rb")
+    if not handle then return nil end
+    local data = handle:read("*a")
+    handle:close()
+    return data
+end
+
+local templates = { contents("/etc/%httpconfigdir/conf.orig/xcat.conf.apach24"),
+                    contents("/etc/%httpconfigdir/conf.orig/xcat.conf.apach22") }
+
+for _, active in ipairs({ "/etc/httpd/conf.d/xcat.conf",
+                          "/etc/apache2/conf.d/xcat.conf" }) do
+    -- A symlink or any other non-regular path is treated as locally managed.
+    if posix.stat(active, "type") == "regular" then
+        local current = contents(active)
+        for _, template in ipairs(templates) do
+            if current and current == template then
+                os.remove(active)
+                break
+            end
+        end
+    end
+end
+%endif
+
 %pre
 # this is now handled by requiring /usr/sbin/dhcpd
 #if [ -e "/etc/SuSE-release" ]; then
@@ -199,8 +237,15 @@ chmod 755 $RPM_BUILD_ROOT/install/postscripts/*
 rm LICENSE.html
 mkdir -p postscripts/hostkeys
 cd -
+# Pick the Apache generation at build time.  Selecting it in the post scriptlet
+# instead rewrites a file rpm has already checksummed, defeating noreplace.
+%if 0%{?fedora} || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1200
+cp %{SOURCE7} $RPM_BUILD_ROOT/etc/httpd/conf.d/xcat.conf
+cp %{SOURCE7} $RPM_BUILD_ROOT/etc/apache2/conf.d/xcat.conf
+%else
 cp %{SOURCE1} $RPM_BUILD_ROOT/etc/httpd/conf.d/xcat.conf
 cp %{SOURCE1} $RPM_BUILD_ROOT/etc/apache2/conf.d/xcat.conf
+%endif
 cp %{SOURCE7} $RPM_BUILD_ROOT/etc/%httpconfigdir/conf.orig/xcat.conf.apach24
 cp %{SOURCE1} $RPM_BUILD_ROOT/etc/%httpconfigdir/conf.orig/xcat.conf.apach22
 cp %{SOURCE5} $RPM_BUILD_ROOT/etc/xCATMN
@@ -211,25 +256,6 @@ cp LICENSE.html $RPM_BUILD_ROOT/%{prefix}/share/doc/packages/xCAT
 
 %post
 %ifos linux
-#Apply the correct httpd/apache configuration file according to the httpd/apache version
-if [ -n "$(httpd -v 2>&1 |grep -e '^Server version\s*:.*\/2.4')" ]
-then
-   rm -rf /etc/httpd/conf.d/xcat.conf
-   cp /etc/%httpconfigdir/conf.orig/xcat.conf.apach24 /etc/httpd/conf.d/xcat.conf
-fi
-
-if [ -n "$(apachectl -v 2>&1 |grep -e '^Server version\s*:.*\/2.4')" ]
-then
-   rm -rf /etc/apache2/conf.d/xcat.conf
-   cp /etc/%httpconfigdir/conf.orig/xcat.conf.apach24 /etc/apache2/conf.d/xcat.conf
-fi
-
-if [ -n "$(apache2ctl -v 2>&1 |grep -e '^Server version\s*:.*\/2.4')" ]
-then
-   rm -rf /etc/apache2/conf.d/xcat.conf
-   cp /etc/%httpconfigdir/conf.orig/xcat.conf.apach24 /etc/apache2/conf.d/xcat.conf
-fi
-
 # On SUSE apache2, mod_headers is not loaded by default; enable it so the
 # security response headers in xcat.conf take effect (a no-op where a2enmod
 # is absent, e.g. httpd on EL where mod_headers is already loaded).
@@ -282,10 +308,9 @@ if grep -q "command returned error code" "$xcatupgradeout"; then
 fi
 rm -f "$xcatupgradeout"
 
-# The package replaces xcat.conf before this upgrade path runs.  Reload an
-# active web server so the new security headers and ServerTokens setting take
-# effect immediately, but do not try to manage services while building an
-# image in a chroot.
+# rpm has either updated a stock xcat.conf or preserved a modified one and left
+# the new file as xcat.conf.rpmnew.  Reload an active web server so an updated
+# stock configuration takes effect, but do not manage services in a chroot.
 if [ -f "/proc/cmdline" ] && [ "x$(stat -c '%i %d' /)" == "x$(stat -c '%i %d' /proc/1/root/. 2>/dev/null)" ]; then
     if [ -e "/etc/redhat-release" ]; then
         apachedaemon='httpd'
@@ -311,8 +336,8 @@ exit 0
 # one for sles, one for rhel. yes, it's ugly...
 /etc/%httpconfigdir/conf.orig/xcat.conf.apach24
 /etc/%httpconfigdir/conf.orig/xcat.conf.apach22
-/etc/httpd/conf.d/xcat.conf
-/etc/apache2/conf.d/xcat.conf
+%config(noreplace) /etc/httpd/conf.d/xcat.conf
+%config(noreplace) /etc/apache2/conf.d/xcat.conf
 /etc/xCATMN
 /install/postscripts
 /install/prescripts

xCATsn/xCATsn.spec

@@ -50,7 +50,6 @@ Conflicts: xCAT
 # on RHEL7, need to specify it explicitly
 Requires: net-tools
 Requires: /usr/bin/killall
-Requires: /usr/bin/bc
 # yaboot-xcat is pulled in so any SN can manage ppc nodes
 Requires: httpd nfs-utils nmap bind
 # DHCP backend resolved at INSTALL time (not build time) via an RPM rich
@@ -112,8 +111,15 @@ mkdir -p $RPM_BUILD_ROOT/etc/apache2/conf.d
 mkdir -p $RPM_BUILD_ROOT/etc/httpd/conf.d/
 mkdir -p $RPM_BUILD_ROOT/%{prefix}/share/xcat/
 # cd -
+# Pick the Apache generation at build time.  Selecting it in the post scriptlet
+# instead rewrites a file rpm has already checksummed, defeating noreplace.
+%if 0%{?fedora} || 0%{?rhel} >= 7 || 0%{?suse_version} >= 1200
+cp %{SOURCE6} $RPM_BUILD_ROOT/etc/apache2/conf.d/xcat.conf
+cp %{SOURCE6} $RPM_BUILD_ROOT/etc/httpd/conf.d/xcat.conf
+%else
 cp %{SOURCE1} $RPM_BUILD_ROOT/etc/apache2/conf.d/xcat.conf
 cp %{SOURCE1} $RPM_BUILD_ROOT/etc/httpd/conf.d/xcat.conf
+%endif
 cp %{SOURCE3} $RPM_BUILD_ROOT/etc/xCATSN
 cp %{SOURCE1} $RPM_BUILD_ROOT/etc/xcat/conf.orig/xcat.conf.apach22
 cp %{SOURCE6} $RPM_BUILD_ROOT/etc/xcat/conf.orig/xcat.conf.apach24
@@ -141,6 +147,44 @@ rm templates.tar
 
 %endif
 
+%ifos linux
+# rpm fixes each config file's fate before %pre runs, so a migration there can
+# delete an active file rpm has already decided to leave as xcat.conf.rpmnew.
+# %pretrans runs before that decision.  It is Lua because a pre-transaction
+# scriptlet cannot rely on any dependency being unpacked yet.
+%pretrans -p <lua>
+-- Older packages shipped xcat.conf as ordinary payload and then rewrote it
+-- from conf.orig in %post, so rpm's recorded digest cannot tell a stock file
+-- from a customised one.  Compare against the outgoing package's templates,
+-- still on disk at this point, and drop only a file that still matches one.
+if arg[2] and tonumber(arg[2]) == 1 then return end
+
+local function contents(path)
+    local handle = io.open(path, "rb")
+    if not handle then return nil end
+    local data = handle:read("*a")
+    handle:close()
+    return data
+end
+
+local templates = { contents("/etc/xcat/conf.orig/xcat.conf.apach24"),
+                    contents("/etc/xcat/conf.orig/xcat.conf.apach22") }
+
+for _, active in ipairs({ "/etc/httpd/conf.d/xcat.conf",
+                          "/etc/apache2/conf.d/xcat.conf" }) do
+    -- A symlink or any other non-regular path is treated as locally managed.
+    if posix.stat(active, "type") == "regular" then
+        local current = contents(active)
+        for _, template in ipairs(templates) do
+            if current and current == template then
+                os.remove(active)
+                break
+            end
+        end
+    end
+end
+%endif
+
 %pre
 # only need to check on AIX
 %ifnos linux
@@ -155,25 +199,6 @@ fi
 
 %post
 %ifos linux
-#Apply the correct httpd/apache configuration file according to the httpd/apache version
-
-compare_version="2.4"
-version=`rpm -qi httpd 2>&1 | grep 'Version' | awk '$NF { print $3 }' | cut -d. -f1,2`
-if [ -n "$version" ]; then
-    if (( `bc <<< "$version >= $compare_version"` )); then
-        rm -rf /etc/httpd/conf.d/xcat.conf
-        cp /etc/xcat/conf.orig/xcat.conf.apach24 /etc/httpd/conf.d/xcat.conf
-    fi
-fi
-
-version=`rpm -qi apache2 2>&1 | grep '^Version' | awk '$NF { print $3 }' | cut -d. -f1,2`
-if [ -n "$version" ]; then
-    if (( `bc <<< "$version >= $compare_version"` )); then
-        rm -rf /etc/apache2/conf.d/xcat.conf
-        cp /etc/xcat/conf.orig/xcat.conf.apach24 /etc/apache2/conf.d/xcat.conf
-    fi
-fi
-
 # On SUSE apache2, mod_headers is not loaded by default; enable it so the
 # security response headers in xcat.conf take effect (a no-op where a2enmod
 # is absent, e.g. httpd on EL where mod_headers is already loaded).
@@ -293,8 +318,8 @@ fi
 %ifos linux
 /etc/xcat/conf.orig/xcat.conf.apach24
 /etc/xcat/conf.orig/xcat.conf.apach22
-/etc/httpd/conf.d/xcat.conf
-/etc/apache2/conf.d/xcat.conf
+%config(noreplace) /etc/httpd/conf.d/xcat.conf
+%config(noreplace) /etc/apache2/conf.d/xcat.conf
 /etc/xcat/logrotate.conf/
 /etc/xcat/rsyslog.conf/
 %endif