diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fa5ea52dbb..8a5e3bb4c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,13 +75,13 @@ jobs: # expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to # test on one big-endian architecture to be sure that checksums are correct. arch: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: include: - - image: '--platform=linux/ppc64le ubuntu:22.04' - - image: '--platform=linux/s390x ubuntu:22.04' + - arch: 'ppc64le' + - arch: 's390x' steps: - name: Checkout Code @@ -89,23 +89,19 @@ jobs: with: path: pgbackrest - - name: Run Test - uses: uraimo/run-on-arch-action@v2 - id: runcmd - with: - base_image: ${{matrix.image}} - - # Cache builds - githubToken: ${{github.token}} + - name: Install + run: | + sudo apt-get update + sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev + docker run --privileged --rm tonistiigi/binfmt --install all - install: | - apt-get update - DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache python3-distutils meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev + - name: Build VM + run: ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm-build --vm=u22 --vm-arch=${{matrix.arch}} - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest - ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --min-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup - ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --min-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface + - name: Run Test + run: | + ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup + ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface # Run meson unity build to check for errors, unused functions, and externed functions unity: diff --git a/test/container.yaml b/test/container.yaml index c4047bc4ba..bddbd8bc19 100644 --- a/test/container.yaml +++ b/test/container.yaml @@ -10,15 +10,19 @@ # # To upload a new image: # - docker login -u pgbackrest -# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?} +# - DATE=YYYYMMDDX;VM=X;ARCH=X;BASE=pgbackrest/test:${VM?}-base-${ARCH?};docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?} # ********************************************************************************************************************************** -20241118A: +20250822A: x86_64: - f41: ce870455184e991e0efd90176da1412f0f3f72a2 - -20241016A: - x86_64: - d11: f3bc523f10e873f85b889120ea08c6c53358cc47 + a321: dab99d91eed1710c68dae28efa5cd0e27e0a7f60 + d11: be044bf120ca3e7d8bed69d5a83b4de4926be87f + f42: f1c2eceeed7b66d8096a9b891f28eb30f141d1b2 rh8: 4d141c845abfbdbf402ba447cf2bd2e4357c8a63 u20: 711ee90bf3ea0e8bb054285193811266b9b1f2f3 - u22: 57c2af91a1b1a9624e325b704cc9213821c6b968 + u22: b7d5253d70ac4c00f615761e2ccc9e24835dd054 + + ppc64le: + u22: cf3637d799de2b3d847f74777c839271b5c8e268 + + s390x: + u22: 131ba5585c13943dd950d5aa025ac73ac7c30767 diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index b987c52882..9ffa752d12 100644 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -84,6 +84,7 @@ sub containerWrite my $oStorageDocker = shift; my $strTempPath = shift; my $strOS = shift; + my $strArch = shift; my $strTitle = shift; my $strImageParent = shift; my $strImage = shift; @@ -104,14 +105,16 @@ sub containerWrite my $strScriptSha1; my $bCached = false; - if ($strImage =~ /\-base$/) + if ($strImage =~ /\-base\-/) { $strScriptSha1 = sha1_hex($strScript); foreach my $strBuild (reverse(keys(%{$hContainerCache}))) { - if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) && - $hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1) + my $strArchLookup = defined($strArch) ? $strArch : hostArch(); + + if (defined($hContainerCache->{$strBuild}{$strArchLookup}{$strOS}) && + $hContainerCache->{$strBuild}{$strArchLookup}{$strOS} eq $strScriptSha1) { &log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ..."); @@ -132,7 +135,8 @@ sub containerWrite # Write the image $oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n"); executeTest( - 'docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " . + 'docker build' . (defined($strArch) ? " --platform linux/${strArch}" : '') . + (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " . $oStorageDocker->pathGet('test'), {bSuppressStdErr => true, bShowOutputAsync => (logLevel())[1] eq DETAIL}); } @@ -338,6 +342,7 @@ sub containerBuild { my $oStorageDocker = shift; my $strVm = shift; + my $strArch = shift; my $bVmForce = shift; # Create temp path @@ -378,8 +383,11 @@ sub containerBuild # Base image ########################################################################################################################### - my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}"; - my $strImage = "${strOS}-base"; + my $strImageParent = + (defined($strArch) ? "${strArch}/" : (vmArch($strOS) eq VM_ARCH_X86_64 ? '' : vmArch($strOS) . '/')) . + "$$oVm{$strOS}{&VM_IMAGE}"; + $strArch = defined($strArch) ? $strArch : hostArch(); + my $strImage = "${strOS}-base" . (defined($strArch) ? "-${strArch}" : '-' . hostArch()); my $strCopy = undef; #--------------------------------------------------------------------------------------------------------------------------- @@ -488,10 +496,10 @@ sub containerBuild if (vmPgRepo($strVm)) { $strScript .= - " echo \"deb http://apt" . ($strVm eq VM_U20 ? '-archive' : '') . ".postgresql.org/pub/repos/apt/ \$(lsb_release -s -c)-pgdg main" . - "\" >> /etc/apt/sources.list.d/pgdg.list && \\\n" . - " wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \\\n" . - " apt-get update && \\\n"; + " apt-get install -y --no-install-recommends postgresql-common && \\\n" . + " /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y" . + ($strOS eq VM_U22 && ($strArch eq VM_ARCH_AARCH64 || $strArch eq VM_ARCH_X86_64) ? ' -c 18' : '') . + " && \\\n"; } $strScript .= @@ -500,7 +508,7 @@ sub containerBuild "/etc/postgresql-common/createcluster.conf"; } - if (defined($oOS->{&VM_DB}) && @{$oOS->{&VM_DB}} > 0) + if (defined($oOS->{&VM_DB}) && @{$oOS->{&VM_DB}} > 0 && ($strArch eq VM_ARCH_AARCH64 || $strArch eq VM_ARCH_X86_64)) { $strScript .= sectionHeader() . "# Install PostgreSQL\n"; @@ -559,14 +567,14 @@ sub containerBuild } containerWrite( - $oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce); + $oStorageDocker, $strTempPath, $strOS, $strArch, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce); # Test image ######################################################################################################################## if (!$bDeprecated) { - $strImageParent = containerRepo() . ":${strOS}-base"; - $strImage = "${strOS}-test"; + $strImageParent = containerRepo() . ":${strImage}"; + $strImage = "${strOS}-test" . (defined($strArch) ? "-${strArch}" : '-' . hostArch()); $strCopy = undef; $strScript = ''; @@ -641,7 +649,8 @@ sub containerBuild $strScript .= entryPointSetup($strOS); containerWrite( - $oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce); + $oStorageDocker, $strTempPath, $strOS, $strArch, 'Test', $strImageParent, $strImage, $strCopy, $strScript, + $bVmForce); } } diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 5e081cd839..4a868f5a2d 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -51,6 +51,8 @@ sub new $self->{oTest}, $self->{bDryRun}, $self->{bVmOut}, + $self->{strPlatform}, + $self->{strImage}, $self->{iVmIdx}, $self->{iVmMax}, $self->{strMakeCmd}, @@ -84,6 +86,8 @@ sub new {name => 'oTest'}, {name => 'bDryRun'}, {name => 'bVmOut'}, + {name => 'strPlatform'}, + {name => 'strImage'}, {name => 'iVmIdx'}, {name => 'iVmMax'}, {name => 'strMakeCmd'}, @@ -207,15 +211,15 @@ sub run my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM}; executeTest( - 'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" . - " -v ${strHostTestPath}:${strVmTestPath}" . + 'docker run' . $self->{strPlatform} . ' -itd -h ' . + $self->{oTest}->{&TEST_VM} . "-test --name=${strImage} -v ${strHostTestPath}:${strVmTestPath}" . ($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') . ($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') . " -v $self->{strBackRestBase}:$self->{strBackRestBase}" . " -v $self->{strRepoPath}:$self->{strRepoPath}" . ($self->{oTest}->{&TEST_C} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') . ($self->{oTest}->{&TEST_C} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') . - ' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test', + ' ' . $self->{strImage}, {bSuppressStdErr => true}); } } diff --git a/test/lib/pgBackRestTest/Common/VmTest.pm b/test/lib/pgBackRestTest/Common/VmTest.pm index 6d6a43cadd..f36142a736 100644 --- a/test/lib/pgBackRestTest/Common/VmTest.pm +++ b/test/lib/pgBackRestTest/Common/VmTest.pm @@ -63,8 +63,8 @@ use constant VM_ARCH_AARCH64 => 'aarch64' push @EXPORT, qw(VM_ARCH_AARCH64); use constant VM_ARCH_I386 => 'i386'; push @EXPORT, qw(VM_ARCH_I386); -use constant VM_ARCH_AMD64 => 'amd64'; - push @EXPORT, qw(VM_ARCH_AMD64); +use constant VM_ARCH_X86_64 => 'x86_64'; + push @EXPORT, qw(VM_ARCH_X86_64); #################################################################################################################################### # Valid VM list @@ -96,7 +96,7 @@ my $oyVm = &VM_NONE => { &VM_OS_BASE => VM_OS_BASE_DEBIAN, - &VM_ARCH => VM_ARCH_AMD64, + &VM_ARCH => VM_ARCH_X86_64, &VMDEF_COVERAGE_C => true, &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', @@ -140,7 +140,7 @@ my $oyVm = { &VM_OS_BASE => VM_OS_BASE_RHEL, &VM_IMAGE => 'rockylinux/rockylinux:8', - &VM_ARCH => VM_ARCH_AMD64, + &VM_ARCH => VM_ARCH_X86_64, &VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin', &VMDEF_DEBUG_INTEGRATION => false, @@ -168,7 +168,7 @@ my $oyVm = { &VM_OS_BASE => VM_OS_BASE_RHEL, &VM_IMAGE => 'fedora:41', - &VM_ARCH => VM_ARCH_AMD64, + &VM_ARCH => VM_ARCH_X86_64, &VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin', &VMDEF_COVERAGE_C => true, @@ -195,7 +195,7 @@ my $oyVm = { &VM_OS_BASE => VM_OS_BASE_DEBIAN, &VM_IMAGE => 'ubuntu:20.04', - &VM_ARCH => VM_ARCH_AMD64, + &VM_ARCH => VM_ARCH_X86_64, &VMDEF_COVERAGE_C => true, &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', @@ -229,7 +229,7 @@ my $oyVm = { &VM_OS_BASE => VM_OS_BASE_DEBIAN, &VM_IMAGE => 'ubuntu:22.04', - &VM_ARCH => VM_ARCH_AMD64, + &VM_ARCH => VM_ARCH_X86_64, &VMDEF_COVERAGE_C => true, &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', @@ -382,6 +382,18 @@ sub vmCoverageC push @EXPORT, qw(vmCoverageC); +#################################################################################################################################### +# Get vm architecture +#################################################################################################################################### +sub vmArch +{ + my $strVm = shift; + + return $oyVm->{$strVm}{&VM_ARCH}; +} + +push @EXPORT, qw(vmArch); + #################################################################################################################################### # Get vm architecture bits #################################################################################################################################### @@ -389,7 +401,7 @@ sub vmArchBits { my $strVm = shift; - return ($oyVm->{$strVm}{&VM_ARCH} eq VM_ARCH_I386 ? 32 : 64); + return (vmArch($strVm) eq VM_ARCH_I386 ? 32 : 64); } push @EXPORT, qw(vmArchBits); @@ -408,7 +420,7 @@ sub hostArch # Mac M1 reports arm64 but we generally need aarch64 (which Linux reports) if ($strHostArch eq 'arm64') { - $strHostArch = 'aarch64'; + $strHostArch = VM_ARCH_AARCH64; } } diff --git a/test/src/common/harnessHost.c b/test/src/common/harnessHost.c index 4e12ce87f4..06aa52a5e9 100644 --- a/test/src/common/harnessHost.c +++ b/test/src/common/harnessHost.c @@ -1093,7 +1093,7 @@ hrnHostBuildRun(const int line, const StringId id) const bool isPg = strBeginsWithZ(name, "pg"); const bool isRepo = id == hrnHostLocal.repoHost; const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name)); - const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm()); + const String *const image = strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()); const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name)); String *const option = strNewFmt( "-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(), @@ -1294,7 +1294,8 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si MEM_CONTEXT_PRIOR_BEGIN() { hrnHostNewP( - HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true); + HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()), + .noUpdateHosts = true); } MEM_CONTEXT_PRIOR_END(); diff --git a/test/test.pl b/test/test.pl index 8c08e8c509..77d754bf68 100755 --- a/test/test.pl +++ b/test/test.pl @@ -108,6 +108,7 @@ =head1 SYNOPSIS VM Options: --vm docker container to build/test (e.g. rh8) + --vm-arch docker container architecture --vm-build build Docker containers --vm-force force a rebuild of Docker containers --vm-out Show VM output (default false) @@ -144,6 +145,7 @@ =head1 SYNOPSIS my $bQuiet = false; my $strPgVersion = 'minimal'; my $strVm = VM_NONE; +my $strVmArch; my $bVmBuild = false; my $bVmForce = false; my $bBuildOnly = false; @@ -185,6 +187,7 @@ =head1 SYNOPSIS 'log-level-test-file=s' => \$strLogLevelTestFile, 'no-log-timestamp' => \$bNoLogTimestamp, 'vm=s' => \$strVm, + 'vm-arch=s' => \$strVmArch, 'vm-out' => \$bVmOut, 'vm-build' => \$bVmBuild, 'vm-force' => \$bVmForce, @@ -375,7 +378,7 @@ =head1 SYNOPSIS ################################################################################################################################ if ($bVmBuild) { - containerBuild($oStorageBackRest, $strVm, $bVmForce); + containerBuild($oStorageBackRest, $strVm, $strVmArch, $bVmForce); exit 0; } @@ -546,6 +549,9 @@ =head1 SYNOPSIS # Start build container if vm is not none #------------------------------------------------------------------------------------------------------------------------------- + my $strPlatform = defined($strVmArch) ? " --platform linux/${strVmArch}" : ''; + my $strImage = containerRepo() . ":${strVm}-test" . (defined($strVmArch) ? "-${strVmArch}" : '-' . hostArch()); + if ($strVm ne VM_NONE) { my $strCCachePath = "${strTestPath}/ccache-0/${strVm}"; @@ -556,9 +562,9 @@ =head1 SYNOPSIS } executeTest( - "docker run -itd -h test-build --name=test-build" . + "docker run${strPlatform} -itd -h test-build --name=test-build" . " -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" . - " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test", + " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . " ${strImage}", {bSuppressStdErr => true}); } @@ -791,8 +797,8 @@ =head1 SYNOPSIS if ($strVm ne VM_NONE) { executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test", + "docker run${strPlatform} -itd -h test-build --name=test-build" . + " -v ${strBackRestBase}:${strBackRestBase} ${strImage}", {bSuppressStdErr => true}); } @@ -875,8 +881,8 @@ =head1 SYNOPSIS if ($strVm ne VM_NONE) { executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test", + "docker run${strPlatform} -itd -h test-build --name=test-build" . + " -v ${strBackRestBase}:${strBackRestBase} ${strImage}", {bSuppressStdErr => true}); } @@ -1055,10 +1061,10 @@ =head1 SYNOPSIS if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun}) { my $oJob = new pgBackRestTest::Common::JobTest( - $oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax, - $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp, - $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary, - !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace, + $oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $strPlatform, + $strImage, $iVmIdx, $iVmMax, $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, + $strLogLevelTestFile, !$bNoLogTimestamp, $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, + !$bNoCoverage, $bCoverageSummary, !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace, $iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax)); $iTestIdx++;