Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 14 additions & 19 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,37 +75,32 @@ jobs:
# expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to
# test on one big-endian architecture to be sure that checksums are correct.
arch:
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04

strategy:
matrix:
include:
- image: '--platform=linux/ppc64le ubuntu:22.04'
- image: '--platform=linux/s390x ubuntu:22.04'

- arch: 'ppc64le'
- arch: 's390x'
steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
path: pgbackrest

- name: Run Test
uses: uraimo/run-on-arch-action@v2
id: runcmd
with:
base_image: ${{matrix.image}}

# Cache builds
githubToken: ${{github.token}}
- name: Install
run: |
sudo apt-get update
sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
docker run --privileged --rm tonistiigi/binfmt --install all

install: |
apt-get update
DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache python3-distutils meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev
- name: Build VM
run: ${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm-build --vm=u22 --vm-arch=${{matrix.arch}}

run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE?}/pgbackrest
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --min-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --min-gen --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface
- name: Run Test
run: |
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=command --test=backup
${GITHUB_WORKSPACE?}/pgbackrest/test/test.pl --vm=u22 --vm-arch=${{matrix.arch}} --no-valgrind --no-coverage --no-optimize --build-max=2 --module=postgres --test=interface

# Run meson unity build to check for errors, unused functions, and externed functions
unity:
Expand Down
15 changes: 9 additions & 6 deletions test/container.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,18 @@
#
# To upload a new image:
# - docker login -u pgbackrest
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# - DATE=YYYYMMDDX;VM=X;ARCH=X;BASE=pgbackrest/test:${VM?}-base-${ARCH?};docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# **********************************************************************************************************************************
20241118A:
x86_64:
f41: ce870455184e991e0efd90176da1412f0f3f72a2
20250228A:
ppc64le:
u22: 28fa02cb370bbdacadf984afb215d3973ed5ab3e

s390x:
u22: f3e108dd7f808f6b0e82eadcfd72f585a7632530

20241016A:
x86_64:
d11: f3bc523f10e873f85b889120ea08c6c53358cc47
d11: 01e6970744c2b2529a14832e92cb861c7da94308
f41: ce870455184e991e0efd90176da1412f0f3f72a2
rh8: 4d141c845abfbdbf402ba447cf2bd2e4357c8a63
u20: 711ee90bf3ea0e8bb054285193811266b9b1f2f3
u22: 57c2af91a1b1a9624e325b704cc9213821c6b968
32 changes: 21 additions & 11 deletions test/lib/pgBackRestTest/Common/ContainerTest.pm
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ sub containerWrite
my $oStorageDocker = shift;
my $strTempPath = shift;
my $strOS = shift;
my $strArch = shift;
my $strTitle = shift;
my $strImageParent = shift;
my $strImage = shift;
Expand All @@ -104,14 +105,16 @@ sub containerWrite
my $strScriptSha1;
my $bCached = false;

if ($strImage =~ /\-base$/)
if ($strImage =~ /\-base\-/)
{
$strScriptSha1 = sha1_hex($strScript);

foreach my $strBuild (reverse(keys(%{$hContainerCache})))
{
if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) &&
$hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1)
my $strArchLookup = defined($strArch) ? $strArch : hostArch();

if (defined($hContainerCache->{$strBuild}{$strArchLookup}{$strOS}) &&
$hContainerCache->{$strBuild}{$strArchLookup}{$strOS} eq $strScriptSha1)
{
&log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ...");

Expand All @@ -132,7 +135,8 @@ sub containerWrite
# Write the image
$oStorageDocker->put("${strTempPath}/${strImage}", trim($strScript) . "\n");
executeTest(
'docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
'docker build' . (defined($strArch) ? " --platform linux/${strArch}" : '') .
(defined($bForce) && $bForce ? ' --no-cache' : '') . " -f ${strTempPath}/${strImage} -t ${strTag} " .
$oStorageDocker->pathGet('test'),
{bSuppressStdErr => true, bShowOutputAsync => (logLevel())[1] eq DETAIL});
}
Expand Down Expand Up @@ -338,6 +342,7 @@ sub containerBuild
{
my $oStorageDocker = shift;
my $strVm = shift;
my $strArch = shift;
my $bVmForce = shift;

# Create temp path
Expand Down Expand Up @@ -378,8 +383,10 @@ sub containerBuild

# Base image
###########################################################################################################################
my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}";
my $strImage = "${strOS}-base";
my $strImageParent =
(defined($strArch) ? "${strArch}/" : (vmArch($strOS) eq VM_ARCH_AMD64 ? '' : vmArch($strOS) . '/')) .
"$$oVm{$strOS}{&VM_IMAGE}";
my $strImage = "${strOS}-base" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());
my $strCopy = undef;

#---------------------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -500,7 +507,9 @@ sub containerBuild
"/etc/postgresql-common/createcluster.conf";
}

if (defined($oOS->{&VM_DB}) && @{$oOS->{&VM_DB}} > 0)
$strScript .= sectionHeader() . "# Arch = " . vmArch($strOS);
if (defined($oOS->{&VM_DB}) && @{$oOS->{&VM_DB}} > 0 &&
vmArch($strOS) eq VM_ARCH_AMD64)
{
$strScript .= sectionHeader() .
"# Install PostgreSQL\n";
Expand Down Expand Up @@ -559,14 +568,14 @@ sub containerBuild
}

containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Base', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);

# Test image
########################################################################################################################
if (!$bDeprecated)
{
$strImageParent = containerRepo() . ":${strOS}-base";
$strImage = "${strOS}-test";
$strImageParent = containerRepo() . ":${strImage}";
$strImage = "${strOS}-test" . (defined($strArch) ? "-${strArch}" : '-' . hostArch());

$strCopy = undef;
$strScript = '';
Expand Down Expand Up @@ -641,7 +650,8 @@ sub containerBuild
$strScript .= entryPointSetup($strOS);

containerWrite(
$oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce);
$oStorageDocker, $strTempPath, $strOS, $strArch, 'Test', $strImageParent, $strImage, $strCopy, $strScript,
$bVmForce);
}
}

Expand Down
10 changes: 7 additions & 3 deletions test/lib/pgBackRestTest/Common/JobTest.pm
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ sub new
$self->{oTest},
$self->{bDryRun},
$self->{bVmOut},
$self->{strPlatform},
$self->{strImage},
$self->{iVmIdx},
$self->{iVmMax},
$self->{strMakeCmd},
Expand Down Expand Up @@ -84,6 +86,8 @@ sub new
{name => 'oTest'},
{name => 'bDryRun'},
{name => 'bVmOut'},
{name => 'strPlatform'},
{name => 'strImage'},
{name => 'iVmIdx'},
{name => 'iVmMax'},
{name => 'strMakeCmd'},
Expand Down Expand Up @@ -207,15 +211,15 @@ sub run
my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM};

executeTest(
'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
" -v ${strHostTestPath}:${strVmTestPath}" .
'docker run' . $self->{strPlatform} . ' -itd -h ' .
$self->{oTest}->{&TEST_VM} . "-test --name=${strImage} -v ${strHostTestPath}:${strVmTestPath}" .
($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') .
($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') .
" -v $self->{strBackRestBase}:$self->{strBackRestBase}" .
" -v $self->{strRepoPath}:$self->{strRepoPath}" .
($self->{oTest}->{&TEST_C} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') .
($self->{oTest}->{&TEST_C} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') .
' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test',
' ' . $self->{strImage},
{bSuppressStdErr => true});
}
}
Expand Down
16 changes: 15 additions & 1 deletion test/lib/pgBackRestTest/Common/VmTest.pm
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ use constant VM_ARCH_I386 => 'i386';
push @EXPORT, qw(VM_ARCH_I386);
use constant VM_ARCH_AMD64 => 'amd64';
push @EXPORT, qw(VM_ARCH_AMD64);
use constant VM_ARCH_X86_64 => 'x86_64';
push @EXPORT, qw(VM_ARCH_X86_64);

####################################################################################################################################
# Valid VM list
Expand Down Expand Up @@ -382,14 +384,26 @@ sub vmCoverageC

push @EXPORT, qw(vmCoverageC);

####################################################################################################################################
# Get vm architecture
####################################################################################################################################
sub vmArch
{
my $strVm = shift;

return $oyVm->{$strVm}{&VM_ARCH};
}

push @EXPORT, qw(vmArch);

####################################################################################################################################
# Get vm architecture bits
####################################################################################################################################
sub vmArchBits
{
my $strVm = shift;

return ($oyVm->{$strVm}{&VM_ARCH} eq VM_ARCH_I386 ? 32 : 64);
return (vmArch($strVm) eq VM_ARCH_I386 ? 32 : 64);
}

push @EXPORT, qw(vmArchBits);
Expand Down
5 changes: 3 additions & 2 deletions test/src/common/harnessHost.c
Original file line number Diff line number Diff line change
Expand Up @@ -1093,7 +1093,7 @@ hrnHostBuildRun(const int line, const StringId id)
const bool isPg = strBeginsWithZ(name, "pg");
const bool isRepo = id == hrnHostLocal.repoHost;
const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name));
const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm());
const String *const image = strNewFmt("pgbackrest/test:%s-test-x86_64", testVm());
const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name));
String *const option = strNewFmt(
"-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(),
Expand Down Expand Up @@ -1294,7 +1294,8 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si
MEM_CONTEXT_PRIOR_BEGIN()
{
hrnHostNewP(
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true);
HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test-x86_64", testVm()),
.noUpdateHosts = true);
}
MEM_CONTEXT_PRIOR_END();

Expand Down
28 changes: 17 additions & 11 deletions test/test.pl
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ =head1 SYNOPSIS

VM Options:
--vm docker container to build/test (e.g. rh8)
--vm-arch docker container architecture
--vm-build build Docker containers
--vm-force force a rebuild of Docker containers
--vm-out Show VM output (default false)
Expand Down Expand Up @@ -144,6 +145,7 @@ =head1 SYNOPSIS
my $bQuiet = false;
my $strPgVersion = 'minimal';
my $strVm = VM_NONE;
my $strVmArch;
my $bVmBuild = false;
my $bVmForce = false;
my $bBuildOnly = false;
Expand Down Expand Up @@ -185,6 +187,7 @@ =head1 SYNOPSIS
'log-level-test-file=s' => \$strLogLevelTestFile,
'no-log-timestamp' => \$bNoLogTimestamp,
'vm=s' => \$strVm,
'vm-arch=s' => \$strVmArch,
'vm-out' => \$bVmOut,
'vm-build' => \$bVmBuild,
'vm-force' => \$bVmForce,
Expand Down Expand Up @@ -375,7 +378,7 @@ =head1 SYNOPSIS
################################################################################################################################
if ($bVmBuild)
{
containerBuild($oStorageBackRest, $strVm, $bVmForce);
containerBuild($oStorageBackRest, $strVm, $strVmArch, $bVmForce);
exit 0;
}

Expand Down Expand Up @@ -546,6 +549,9 @@ =head1 SYNOPSIS

# Start build container if vm is not none
#-------------------------------------------------------------------------------------------------------------------------------
my $strPlatform = defined($strVmArch) ? " --platform linux/${strVmArch}" : '';
my $strImage = containerRepo() . ":${strVm}-test" . (defined($strVmArch) ? "-${strVmArch}" : '-' . hostArch());

if ($strVm ne VM_NONE)
{
my $strCCachePath = "${strTestPath}/ccache-0/${strVm}";
Expand All @@ -556,9 +562,9 @@ =head1 SYNOPSIS
}

executeTest(
"docker run -itd -h test-build --name=test-build" .
"docker run${strPlatform} -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" .
" -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test",
" -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . " ${strImage}",
{bSuppressStdErr => true});
}

Expand Down Expand Up @@ -791,8 +797,8 @@ =head1 SYNOPSIS
if ($strVm ne VM_NONE)
{
executeTest(
"docker run -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
"docker run${strPlatform} -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
{bSuppressStdErr => true});
}

Expand Down Expand Up @@ -875,8 +881,8 @@ =head1 SYNOPSIS
if ($strVm ne VM_NONE)
{
executeTest(
"docker run -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test",
"docker run${strPlatform} -itd -h test-build --name=test-build" .
" -v ${strBackRestBase}:${strBackRestBase} ${strImage}",
{bSuppressStdErr => true});
}

Expand Down Expand Up @@ -1055,10 +1061,10 @@ =head1 SYNOPSIS
if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun})
{
my $oJob = new pgBackRestTest::Common::JobTest(
$oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax,
$strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp,
$bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary,
!$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
$oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $strPlatform,
$strImage, $iVmIdx, $iVmMax, $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest,
$strLogLevelTestFile, !$bNoLogTimestamp, $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind,
!$bNoCoverage, $bCoverageSummary, !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace,
$iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax));
$iTestIdx++;

Expand Down
Loading