Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 14 additions & 18 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,37 +75,33 @@ jobs:
# expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to
# test on one big-endian architecture to be sure that checksums are correct.
arch:
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04

strategy:
matrix:
include:
- image: '--platform=linux/ppc64le ubuntu:22.04'
- image: '--platform=linux/s390x ubuntu:22.04'
- arch: 'ppc64le'
- arch: 's390x'

steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
path: pgbackrest

- name: Run Test
uses: uraimo/run-on-arch-action@v2
id: runcmd
with:
base_image: ${{matrix.image}}

# Cache builds
githubToken: ${{github.token}}
- name: Install
run: |
sudo apt-get update
sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
docker run --privileged --rm tonistiigi/binfmt --install all

install: |
apt-get update
DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache python3-distutils meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
- name: Build VM
run: ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm-build --vm=u22 --vm-arch=${{matrix.arch}}

run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --min-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --min-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
- name: Run Test
run: |
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface

# Run meson unity build to check for errors, unused functions, and externed functions
unity:
Expand Down
20 changes: 12 additions & 8 deletions test/container.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,19 @@
#
# To upload a new image:
# - docker login -u pgbackrest
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# - DATE=YYYYMMDDX;VM=X;ARCH=X;BASE=pgbackrest/test:${VM?}-base-${ARCH?};docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# **********************************************************************************************************************************
20241118A:
20250822A:
x86_64:
f41: ce870455184e991e0efd90176da1412f0f3f72a2

20241016A:
x86_64:
d11: f3bc523f10e873f85b889120ea08c6c53358cc47
a321: dab99d91eed1710c68dae28efa5cd0e27e0a7f60
d11: be044bf120ca3e7d8bed69d5a83b4de4926be87f
f42: f1c2eceeed7b66d8096a9b891f28eb30f141d1b2
rh8: 4d141c845abfbdbf402ba447cf2bd2e4357c8a63
u20: 711ee90bf3ea0e8bb054285193811266b9b1f2f3
u22: 57c2af91a1b1a9624e325b704cc9213821c6b968
u22: b7d5253d70ac4c00f615761e2ccc9e24835dd054

ppc64le:
u22: cf3637d799de2b3d847f74777c839271b5c8e268

s390x:
u22: 131ba5585c13943dd950d5aa025ac73ac7c30767
39 changes: 24 additions & 15 deletions test/lib/pgBackRestTest/Common/ContainerTest.pm
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ sub containerWrite
my $oStorageDocker = shift;
my $strTempPath = shift;
my $strOS = shift;
my $strArch = shift;
my $strTitle = shift;
my $strImageParent = shift;
my $strImage = shift;
Expand All @@ -104,14 +105,16 @@ sub containerWrite
my $strScriptSha1;
my $bCached = false;

if ($strImage =~ /\-base$/)
if ($strImage =~ /\-base\-/)
{
$strScriptSha1 = sha1_hex($strScript);

foreach my $strBuild (reverse(keys(%{$hContainerCache})))
{
if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) &&
$hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1)
my $strArchLookup = defined($strArch) ? $strArch : hostArch();

if (defined($hContainerCache->{$strBuild}{$strArchLookup}{$strOS}) &&
$hContainerCache->{$strBuild}{$strArchLookup}{$strOS} eq $strScriptSha1)
{
&log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ...");

Expand All @@ -132,7 +135,8 @@ sub containerWrite
# Write the image
$oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n");
executeTest(
'docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
'docker build' . (defined($strArch) ? " --platform linux/${strArch}" : '') .
(defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
$oStorageDocker->pathGet('test'),
{bSuppressStdErr => true, bShowOutputAsync => (logLevel())[1] eq DETAIL});
}
Expand Down Expand Up @@ -338,6 +342,7 @@ sub containerBuild
{
my $oStorageDocker = shift;
my $strVm = shift;
my $strArch = shift;
my $bVmForce = shift;

# Create temp path
Expand Down Expand Up @@ -378,8 +383,11 @@ sub containerBuild

# Base image
###########################################################################################################################
my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}";
my $strImage = "${strOS}-base";
my $strImageParent =
(defined($strArch) ? "${strArch}/" : (vmArch($strOS) eq VM_ARCH_X86_64 ? '' : vmArch($strOS) . '/')) .
"$$oVm{$strOS}{&VM_IMAGE}";
$strArch = defined($strArch) ? $strArch : hostArch();
my $strImage = "${strOS}-base" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
my $strCopy = undef;

#---------------------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -488,10 +496,10 @@ sub containerBuild
if (vmPgRepo($strVm))
{
$strScript .=
" echo \"deb http://apt" . ($strVm eq VM_U20 ? '-archive' : '') . ".postgresql.org/pub/repos/apt/ \$(lsb_release -s -c)-pgdg main" .
"\" >> /etc/apt/sources.list.d/pgdg.list && \\\n" .
" wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \\\n" .
" apt-get update && \\\n";
" apt-get install -y --no-install-recommends postgresql-common && \\\n" .
" /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y" .
($strOS eq VM_U22 && ($strArch eq VM_ARCH_AARCH64 || $strArch eq VM_ARCH_X86_64) ? ' -c 18' : '') .
" && \\\n";
}

$strScript .=
Expand All @@ -500,7 +508,7 @@ sub containerBuild
"/etc/postgresql-common/createcluster.conf";
}

if (defined($oOS->{&VM_DB}) && @{$oOS->{&VM_DB}} > 0)
if (defined($oOS->{&VM_DB}) && @{$oOS->{&VM_DB}} > 0 && ($strArch eq VM_ARCH_AARCH64 || $strArch eq VM_ARCH_X86_64))
{
$strScript .= sectionHeader() .
"# Install PostgreSQL\n";
Expand Down Expand Up @@ -559,14 +567,14 @@ sub containerBuild
}

containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);

# Test image
########################################################################################################################
if (!$bDeprecated)
{
$strImageParent = containerRepo() . ":${strOS}-base";
$strImage = "${strOS}-test";
$strImageParent = containerRepo() . ":${strImage}";
$strImage = "${strOS}-test" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());

$strCopy = undef;
$strScript = '';
Expand Down Expand Up @@ -641,7 +649,8 @@ sub containerBuild
$strScript .= entryPointSetup($strOS);

containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Test', $strImageParent, $strImage, $strCopy, $strScript,
$bVmForce);
}
}

Expand Down
10 changes: 7 additions & 3 deletions test/lib/pgBackRestTest/Common/JobTest.pm
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ sub new
$self->{oTest},
$self->{bDryRun},
$self->{bVmOut},
$self->{strPlatform},
$self->{strImage},
$self->{iVmIdx},
$self->{iVmMax},
$self->{strMakeCmd},
Expand Down Expand Up @@ -84,6 +86,8 @@ sub new
{name => 'oTest'},
{name => 'bDryRun'},
{name => 'bVmOut'},
{name => 'strPlatform'},
{name => 'strImage'},
{name => 'iVmIdx'},
{name => 'iVmMax'},
{name => 'strMakeCmd'},
Expand Down Expand Up @@ -207,15 +211,15 @@ sub run
my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM};

executeTest(
'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
" -v ${strHostTestPath}:${strVmTestPath}" .
'docker run' . $self->{strPlatform} . ' -itd -h ' .
$self->{oTest}->{&TEST_VM} . "-test --name=${strImage} -v ${strHostTestPath}:${strVmTestPath}" .
($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') .
($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') .
" -v $self->{strBackRestBase}:$self->{strBackRestBase}" .
" -v $self->{strRepoPath}:$self->{strRepoPath}" .
($self->{oTest}->{&TEST_C} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') .
($self->{oTest}->{&TEST_C} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') .
' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test',
' ' . $self->{strImage},
{bSuppressStdErr => true});
}
}
Expand Down
30 changes: 21 additions & 9 deletions test/lib/pgBackRestTest/Common/VmTest.pm
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ use constant VM_ARCH_AARCH64 => 'aarch64'
push @EXPORT, qw(VM_ARCH_AARCH64);
use constant VM_ARCH_I386 => 'i386';
push @EXPORT, qw(VM_ARCH_I386);
use constant VM_ARCH_AMD64 => 'amd64';
push @EXPORT, qw(VM_ARCH_AMD64);
use constant VM_ARCH_X86_64 => 'x86_64';
push @EXPORT, qw(VM_ARCH_X86_64);

####################################################################################################################################
# Valid VM list
Expand Down Expand Up @@ -96,7 +96,7 @@ my $oyVm =
&VM_NONE =>
{
&VM_OS_BASE => VM_OS_BASE_DEBIAN,
&VM_ARCH => VM_ARCH_AMD64,
&VM_ARCH => VM_ARCH_X86_64,
&VMDEF_COVERAGE_C => true,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',

Expand Down Expand Up @@ -140,7 +140,7 @@ my $oyVm =
{
&VM_OS_BASE => VM_OS_BASE_RHEL,
&VM_IMAGE => 'rockylinux/rockylinux:8',
&VM_ARCH => VM_ARCH_AMD64,
&VM_ARCH => VM_ARCH_X86_64,
&VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',

&VMDEF_DEBUG_INTEGRATION => false,
Expand Down Expand Up @@ -168,7 +168,7 @@ my $oyVm =
{
&VM_OS_BASE => VM_OS_BASE_RHEL,
&VM_IMAGE => 'fedora:41',
&VM_ARCH => VM_ARCH_AMD64,
&VM_ARCH => VM_ARCH_X86_64,
&VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
&VMDEF_COVERAGE_C => true,

Expand All @@ -195,7 +195,7 @@ my $oyVm =
{
&VM_OS_BASE => VM_OS_BASE_DEBIAN,
&VM_IMAGE => 'ubuntu:20.04',
&VM_ARCH => VM_ARCH_AMD64,
&VM_ARCH => VM_ARCH_X86_64,
&VMDEF_COVERAGE_C => true,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',

Expand Down Expand Up @@ -229,7 +229,7 @@ my $oyVm =
{
&VM_OS_BASE => VM_OS_BASE_DEBIAN,
&VM_IMAGE => 'ubuntu:22.04',
&VM_ARCH => VM_ARCH_AMD64,
&VM_ARCH => VM_ARCH_X86_64,
&VMDEF_COVERAGE_C => true,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',

Expand Down Expand Up @@ -382,14 +382,26 @@ sub vmCoverageC

push @EXPORT, qw(vmCoverageC);

####################################################################################################################################
# Get vm architecture
####################################################################################################################################
sub vmArch
{
my $strVm = shift;

return $oyVm->{$strVm}{&VM_ARCH};
}

push @EXPORT, qw(vmArch);

####################################################################################################################################
# Get vm architecture bits
####################################################################################################################################
sub vmArchBits
{
my $strVm = shift;

return ($oyVm->{$strVm}{&VM_ARCH} eq VM_ARCH_I386 ? 32 : 64);
return (vmArch($strVm) eq VM_ARCH_I386 ? 32 : 64);
}

push @EXPORT, qw(vmArchBits);
Expand All @@ -408,7 +420,7 @@ sub hostArch
# Mac M1 reports arm64 but we generally need aarch64 (which Linux reports)
if ($strHostArch eq 'arm64')
{
$strHostArch = 'aarch64';
$strHostArch = VM_ARCH_AARCH64;
}
}

Expand Down
5 changes: 3 additions & 2 deletions test/src/common/harnessHost.c
Original file line number Diff line number Diff line change
Expand Up @@ -1093,7 +1093,7 @@ hrnHostBuildRun(const int line, const StringId id)
const bool isPg = strBeginsWithZ(name, "pg");
const bool isRepo = id == hrnHostLocal.repoHost;
const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name));
const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm());
const String *const image = strNewFmt("pgbackrest/test:%s-test-x86_64", testVm());
const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name));
String *const option = strNewFmt(
"-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(),
Expand Down Expand Up @@ -1294,7 +1294,8 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si
MEM_CONTEXT_PRIOR_BEGIN()
{
hrnHostNewP(
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true);
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()),
.noUpdateHosts = true);
}
MEM_CONTEXT_PRIOR_END();

Expand Down
Loading
Loading