From 6c21b4dd50cd3aa727c89f8f22ff40d6cae12deb Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 07:47:10 +1000 Subject: [PATCH 01/15] testsuite: add depth/cross-dir/daemon coverage helpers to rsyncfns.py Add helpers for the option-coverage expansion (the path-handling restructure changes parent-component resolution, so options must be exercised at depth and across directory boundaries): * make_tree() builds a tree with a regular file at every level so a property can be checked at the tree root and >=3 levels deep; * walk_files()/walk_dirs() iterate entries for per-level assertions; * assert_same/assert_mode/assert_mtime_close/assert_is_symlink/ assert_hardlinked/assert_not_hardlinked/assert_exists/assert_not_exists assert the concrete property an option controls (not just dest == src); * write_daemon_conf() writes an arbitrary rsyncd.conf (globals + modules) for daemon-parameter tests, beyond build_rsyncd_conf's fixed four modules; * forced_protocol() lets protocol-sensitive tests gate sub-cases. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/rsyncfns.py | 200 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/testsuite/rsyncfns.py b/testsuite/rsyncfns.py index f151fd5e1..b9bcbf17b 100644 --- a/testsuite/rsyncfns.py +++ b/testsuite/rsyncfns.py @@ -19,6 +19,7 @@ import atexit import fcntl +import filecmp import os import platform import shlex @@ -329,6 +330,17 @@ def rsync_argv(*args: str) -> list: return shlex.split(RSYNC) + list(args) +def forced_protocol(): + """The protocol version pinned via --protocol=N in the RSYNC command, or + None when the run isn't pinning one (so the binary negotiates its newest). + Protocol-sensitive tests use this to gate sub-cases -- e.g. the split + between --append and --append-verify only exists at protocol >= 30; at + protocol 29 plain --append behaves like the old verifying append.""" + import re + m = re.search(r'--protocol[ =](\d+)', RSYNC) + return int(m.group(1)) if m else None + + def run_rsync(*args: str, check: bool = True, capture_output: bool = False) -> subprocess.CompletedProcess: """Run rsync with the given arguments. @@ -980,3 +992,191 @@ def check_perms(path, expected: str) -> 'None': print(f"permissions: {perms} on {path}") print(f"should be: {expected}") test_fail(f"check_perms failed for {path}") + + +# --- depth / cross-dir coverage helpers ------------------------------------ +# Added for the option-coverage expansion (see testsuite/COVERAGE.md). +# The path-handling restructure changes how parent components resolve, so its +# bugs surface only at DEPTH and across directory boundaries -- these helpers +# build trees with an entry at every level and assert the concrete property an +# option controls (not just dest == src). + +def make_tree(root, depth: int = 3, *, data: bool = False, + content_lines: int = 20, data_size: int = 4096, + dirname: str = 'd', leaf: str = 'f'): + """Create a layered directory tree with one regular file at every level. + + For depth=3 under `root`: + root/f0 + root/d1/f1 + root/d1/d2/f2 + root/d1/d2/d3/f3 + so an option's effect can be checked at the tree root AND >=3 levels deep + (the parent-component resolution the path restructure rewrites). + + Returns (dirs, files): `dirs` the created subdirectories outermost-first, + `files` the regular files shallow-first. Content is deterministic + (make_text_file) unless data=True (make_data_file, delta-friendly). + """ + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + dirs = [] + files = [] + cur = root + for level in range(depth + 1): + f = cur / f'{leaf}{level}' + if data: + make_data_file(f, data_size) + else: + make_text_file(f, content_lines) + files.append(f) + if level < depth: + cur = cur / f'{dirname}{level + 1}' + cur.mkdir(exist_ok=True) + dirs.append(cur) + return dirs, files + + +def walk_files(root) -> list: + """Every regular (non-symlink) file under `root`, sorted, recursively. + For asserting a per-entry property holds at every depth.""" + root = Path(root) + return sorted(p for p in root.rglob('*') + if p.is_file() and not p.is_symlink()) + + +def walk_dirs(root) -> list: + """Every subdirectory under `root`, sorted, recursively.""" + root = Path(root) + return sorted(p for p in root.rglob('*') + if p.is_dir() and not p.is_symlink()) + + +def _tag(label: str) -> str: + return f"{label}: " if label else "" + + +def assert_same(a, b, label: str = '') -> 'None': + """Fail unless files `a` and `b` have byte-identical content.""" + if not filecmp.cmp(str(a), str(b), shallow=False): + test_fail(f"{_tag(label)}content differs between {a} and {b}") + + +def assert_mode(path, expected_octal: int, label: str = '') -> 'None': + """Fail unless the permission bits (low 12) of `path` equal expected_octal + (pass an int like 0o644). Does not follow symlinks.""" + mode = stat.S_IMODE(os.stat(path, follow_symlinks=False).st_mode) + if mode != expected_octal: + test_fail(f"{_tag(label)}mode {mode:04o} != expected " + f"{expected_octal:04o} on {path}") + + +def assert_mtime_close(a, b, tol: float = 1.0, label: str = '') -> 'None': + """Fail unless the mtimes of `a` and `b` are within `tol` seconds. + `b` may be a number (an explicit epoch mtime) instead of a path.""" + ma = os.stat(a, follow_symlinks=False).st_mtime + mb = b if isinstance(b, (int, float)) else os.stat( + b, follow_symlinks=False).st_mtime + if abs(ma - mb) > tol: + test_fail(f"{_tag(label)}mtime {ma} vs {mb} differ by > {tol}s " + f"(checking {a})") + + +def assert_is_symlink(path, target: str = None, label: str = '') -> 'None': + """Fail unless `path` is a symlink (optionally pointing exactly at + `target`).""" + if not os.path.islink(path): + test_fail(f"{_tag(label)}{path} is not a symlink") + if target is not None: + actual = os.readlink(path) + if actual != target: + test_fail(f"{_tag(label)}{path} -> {actual!r}, " + f"expected {target!r}") + + +def assert_hardlinked(a, b, label: str = '') -> 'None': + """Fail unless `a` and `b` are the same inode (a hard link / --link-dest + result).""" + sa = os.stat(a, follow_symlinks=False) + sb = os.stat(b, follow_symlinks=False) + if (sa.st_dev, sa.st_ino) != (sb.st_dev, sb.st_ino): + test_fail(f"{_tag(label)}{a} and {b} are not hard-linked " + f"(ino {sa.st_ino} vs {sb.st_ino})") + + +def assert_not_hardlinked(a, b, label: str = '') -> 'None': + """Fail if `a` and `b` share an inode (e.g. --copy-dest must copy, not + link).""" + sa = os.stat(a, follow_symlinks=False) + sb = os.stat(b, follow_symlinks=False) + if (sa.st_dev, sa.st_ino) == (sb.st_dev, sb.st_ino): + test_fail(f"{_tag(label)}{a} and {b} unexpectedly share " + f"inode {sa.st_ino}") + + +def assert_exists(path, label: str = '') -> 'None': + """Fail unless `path` exists (a symlink counts even if dangling).""" + if not os.path.lexists(path): + test_fail(f"{_tag(label)}{path} does not exist") + + +def assert_not_exists(path, label: str = '') -> 'None': + """Fail if `path` exists (a dangling symlink counts as existing).""" + if os.path.lexists(path): + test_fail(f"{_tag(label)}{path} exists but should not") + + +def write_daemon_conf(modules, globals=None, *, + name: str = 'test-rsyncd.conf') -> 'Path': + """Write a custom rsyncd.conf for daemon-parameter tests. + + `modules` is a list of (module_name, {param: value}) pairs; `globals` an + optional dict of global parameters that override the minimal defaults + (pid file / use chroot=no / hosts allow / log file / max verbosity). + Mirrors build_rsyncd_conf()'s root-aware uid/gid handling (only emitted + when running as root) and writes the same `ignore23` wrapper, but lets a + test set arbitrary parameters/modules beyond the fixed four. Returns the + config path; pair with start_test_daemon(). + """ + conf = SCRATCHDIR / name + pidfile = SCRATCHDIR / 'rsyncd.pid' + logfile = SCRATCHDIR / 'rsyncd.log' + + g = { + 'pid file': str(pidfile), + 'use chroot': 'no', + 'hosts allow': 'localhost 127.0.0.0/8', + 'log file': str(logfile), + 'max verbosity': '4', + } + if globals: + g.update(globals) + if get_testuid() == get_rootuid(): + g.setdefault('uid', str(get_rootuid())) + g.setdefault('gid', str(get_rootgid())) + else: + # Non-root cannot set uid/gid in rsyncd.conf. + g.pop('uid', None) + g.pop('gid', None) + + lines = ['# autogenerated by rsyncfns.write_daemon_conf', ''] + lines += [f'{k} = {v}' for k, v in g.items()] + lines.append('') + for mod_name, params in modules: + lines.append(f'[{mod_name}]') + lines += [f'\t{k} = {v}' for k, v in params.items()] + lines.append('') + conf.write_text('\n'.join(lines) + '\n') + + ignore23 = SCRATCHDIR / 'ignore23' + if not ignore23.exists(): + ignore23.write_text( + '#!/bin/sh\n' + 'if "${@}"; then exit; fi\n' + 'ret=$?\n' + 'if test $ret = 23; then exit; fi\n' + 'exit $ret\n' + ) + ignore23.chmod(0o755) + + return conf From 5ea1b3e52487d459b2d554d92fbf080e846ad151 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 07:47:10 +1000 Subject: [PATCH 02/15] testsuite: cross-directory/temp/backup/dest coverage at depth Fill the highest-restructure-risk gap: options that do two-directory / rename / outside-tree work, asserted at >=3 levels deep with the aux tree kept outside the main tree, and asserting the option's specific property rather than just tree equality (which the ported tests already cover). alt-dest-deep --link-dest hardlinks unchanged files (same inode), --copy-dest copies (never links), --compare-dest omits unchanged files; ref tree outside both src and dest. temp-dir cross-dir temp->final rename at depth; temp dir left clean; a missing --temp-dir fails (so the option is proven consulted). partial --partial keeps the partial in the dest file; relative --partial-dir stages per-directory at depth (pre-seed + interrupt/resume); absolute --partial-dir writes the partial outside the tree. inplace --inplace keeps the destination inode across a delta update; the default temp+rename path replaces it. append --append completes truncated files tail-only; --append-verify repairs a corrupted prefix (protocol >= 30). backup-deep --suffix saves S beside the new file; --backup-dir relocates old files to a parallel deep tree outside the dest and captures deletions under --delete. All green on master and under --protocol=29/30. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/alt-dest-deep_test.py | 85 ++++++++++++++++++ testsuite/append_test.py | 73 +++++++++++++++ testsuite/backup-deep_test.py | 86 ++++++++++++++++++ testsuite/delay-updates-deep_test.py | 62 +++++++++++++ testsuite/inplace_test.py | 71 +++++++++++++++ testsuite/partial_test.py | 130 +++++++++++++++++++++++++++ testsuite/temp-dir_test.py | 61 +++++++++++++ 7 files changed, 568 insertions(+) create mode 100644 testsuite/alt-dest-deep_test.py create mode 100644 testsuite/append_test.py create mode 100644 testsuite/backup-deep_test.py create mode 100644 testsuite/delay-updates-deep_test.py create mode 100644 testsuite/inplace_test.py create mode 100644 testsuite/partial_test.py create mode 100644 testsuite/temp-dir_test.py diff --git a/testsuite/alt-dest-deep_test.py b/testsuite/alt-dest-deep_test.py new file mode 100644 index 000000000..1054948e8 --- /dev/null +++ b/testsuite/alt-dest-deep_test.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Property-level coverage of --link-dest / --copy-dest / --compare-dest at +depth and across directory boundaries. + +The existing alt-dest_test.py is a faithful port that checks tree equality; +this companion asserts the *distinguishing* property of each option, at every +level of a >=3-deep tree, with the alternate-destination tree placed OUTSIDE +both the source and destination trees (a sibling, not a parent/child): + + --link-dest unchanged files are HARD-LINKED to the reference (same inode); + a changed file is transferred fresh (not linked). + --copy-dest unchanged files are COPIED from the reference (never linked); + dest is complete and byte-identical to the source. + --compare-dest unchanged files are NEITHER transferred NOR created in dest; + only a changed/new file lands in dest. + +These options drive the two-dirfd / outside-tree path handling the resolver +restructure rewrites, so they must be guarded at depth. +""" + +import os + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + assert_exists, assert_hardlinked, assert_not_exists, assert_not_hardlinked, + assert_same, make_tree, rmtree, run_rsync, walk_files, +) + +src = FROMDIR +ref = SCRATCHDIR / 'altref' # sibling of from/ and to/ -- outside both trees + +rmtree(src) +rmtree(ref) +rmtree(TODIR) + +# A >=3-deep source: f0 at the root, then d1/f1, d1/d2/f2, d1/d2/d3/f3. +make_tree(src, depth=3, data=True) + +# Reference tree == an exact copy of the source, so every file is "identical" +# for the alt-dest comparison. +run_rsync('-a', f'{src}/', f'{ref}/') + +# Now make the deepest file differ so it must be transferred in every mode. +changed = os.path.join('d1', 'd2', 'd3', 'f3') +with open(src / changed, 'ab') as f: + f.write(b'a changed deep tail\n') + +rels = [p.relative_to(src) for p in walk_files(src)] +assert os.path.join('d1', 'd2', 'd3', 'f3') in [str(r) for r in rels] + + +def run_to(opt): + rmtree(TODIR) + run_rsync('-a', f'--{opt}={ref}', f'{src}/', f'{TODIR}/') + + +# --- --link-dest: unchanged -> hardlink to ref; changed -> fresh copy ------- +run_to('link-dest') +for rel in rels: + d, r = TODIR / rel, ref / rel + if str(rel) == changed: + assert_not_hardlinked(d, r, label=f'link-dest changed {rel}') + assert_same(d, src / rel, label=f'link-dest changed {rel}') + else: + assert_hardlinked(d, r, label=f'link-dest unchanged {rel}') + +# --- --copy-dest: every file copied (never linked), dest complete ----------- +run_to('copy-dest') +for rel in rels: + d, r = TODIR / rel, ref / rel + assert_exists(d, label=f'copy-dest {rel}') + assert_same(d, src / rel, label=f'copy-dest {rel}') + assert_not_hardlinked(d, r, label=f'copy-dest {rel}') + +# --- --compare-dest: unchanged absent from dest; only the changed file lands - +run_to('compare-dest') +for rel in rels: + d = TODIR / rel + if str(rel) == changed: + assert_exists(d, label=f'compare-dest changed {rel}') + assert_same(d, src / rel, label=f'compare-dest changed {rel}') + else: + assert_not_exists(d, label=f'compare-dest unchanged {rel}') + +print("alt-dest-deep: link-dest/copy-dest/compare-dest verified at depth") diff --git a/testsuite/append_test.py b/testsuite/append_test.py new file mode 100644 index 000000000..46c78338e --- /dev/null +++ b/testsuite/append_test.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Coverage of --append and --append-verify at depth. + +--append assumes each destination file is a prefix of the (longer) source and +transfers only the bytes past the existing size; it does NOT re-check the data +already present, and it never touches a file that is already the same size or +larger. --append-verify works the same way but folds the existing data into the +whole-file checksum, so a transfer whose result fails verification is re-sent +with a normal --inplace pass. Exercise both on files >=3 levels deep. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, forced_protocol, make_tree, rmtree, run_rsync, test_fail, + walk_files, +) + +src = FROMDIR +deep = os.path.join('d1', 'd2', 'd3', 'f3') + + +def seed_source(): + rmtree(src) + make_tree(src, depth=3, data=True, data_size=8192) + return [p.relative_to(src) for p in walk_files(src)] + + +def dest_prefix(rels, *, corrupt=False, frac=0.5): + """Build a destination holding the first `frac` of each source file (a + valid prefix), optionally corrupting the deep file's leading bytes.""" + rmtree(TODIR) + for rel in rels: + dst = TODIR / rel + dst.parent.mkdir(parents=True, exist_ok=True) + full = (src / rel).read_bytes() + dst.write_bytes(full[: int(len(full) * frac)]) + if corrupt: + p = TODIR / deep + bad = bytearray(p.read_bytes()) + bad[0:64] = b'\x00' * 64 + p.write_bytes(bytes(bad)) + + +# --- --append completes truncated destinations at every level --------------- +rels = seed_source() +dest_prefix(rels) +run_rsync('-a', '--append', f'{src}/', f'{TODIR}/') +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'append {rel}') + +# The split between non-verifying --append and verifying --append-verify only +# exists at protocol >= 30; at protocol 29 plain --append still verifies, so +# skip the distinguishing sub-cases there. +proto = forced_protocol() +if proto is not None and proto < 30: + print(f"append: protocol {proto} -- skipping the --append/--append-verify " + "split (verifying-append behaviour predates the protocol-30 split)") +else: + # plain --append trusts a corrupted prefix (leaves it wrong) + dest_prefix(rels, corrupt=True) + run_rsync('-a', '--append', f'{src}/', f'{TODIR}/') + if (TODIR / deep).read_bytes() == (src / deep).read_bytes(): + test_fail("plain --append unexpectedly repaired a corrupted prefix " + "(it should append only and trust the existing data)") + + # --append-verify detects the bad prefix and re-sends the whole file + dest_prefix(rels, corrupt=True) + run_rsync('-a', '--append-verify', f'{src}/', f'{TODIR}/') + assert_same(TODIR / deep, src / deep, label='append-verify deep') + +print("append: tail-only completion at depth; append-verify repairs prefix") diff --git a/testsuite/backup-deep_test.py b/testsuite/backup-deep_test.py new file mode 100644 index 000000000..f2f754c13 --- /dev/null +++ b/testsuite/backup-deep_test.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Property-level coverage of --backup / --suffix / --backup-dir at depth. + +backup_test.py is the ported regression test (2 levels, no custom suffix); this +companion checks the concrete outcome of each backup mode at >=3 levels and, +for --backup-dir, with the backup tree placed OUTSIDE the destination (a +sibling) so the old file is renamed across directories into a parallel deep +path -- the cross-directory operation the resolver restructure rewrites. + +Asserts, at every level of the tree: + * --backup --suffix=S saves the overwritten file as S beside the new + one (old content in the backup, new content in place); + * --backup --backup-dir=DIR relocates the old file to DIR/, + preserving the deep structure, while the destination gets the new content; + * --backup-dir together with --delete routes a deletion into the backup + tree instead of losing it. +""" + +import os + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + assert_not_exists, assert_same, make_tree, rmtree, run_rsync, test_fail, + walk_files, +) + +src = FROMDIR +bak = SCRATCHDIR / 'backups' # sibling of from/ and to/ -- outside both trees + + +def seed(): + """Fresh v1 source, a matching destination, and the old (v1) bytes; then + mutate the source to v2 so the next sync overwrites every file.""" + rmtree(src) + rmtree(TODIR) + rmtree(bak) + make_tree(src, depth=3, data=True, data_size=4096) + rels = [p.relative_to(src) for p in walk_files(src)] + run_rsync('-a', f'{src}/', f'{TODIR}/') + old = {rel: (src / rel).read_bytes() for rel in rels} + for rel in rels: # v1 -> v2 + with open(src / rel, 'ab') as f: + f.write(b'\nversion-2 tail\n') + return rels, old + + +# --- --backup --suffix=.old (same directory) -------------------------------- +rels, old = seed() +run_rsync('-a', '-b', '--suffix=.old', '--no-whole-file', + f'{src}/', f'{TODIR}/') +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'suffix new {rel}') + backup = (TODIR / rel) + backup = backup.with_name(backup.name + '.old') + if not backup.is_file(): + test_fail(f"--suffix backup missing for {rel}: {backup}") + if backup.read_bytes() != old[rel]: + test_fail(f"--suffix backup of {rel} does not hold the old content") + +# --- --backup-dir at depth, outside the dest tree (cross-dir) --------------- +rels, old = seed() +run_rsync('-a', '-b', f'--backup-dir={bak}', '--no-whole-file', + f'{src}/', f'{TODIR}/') +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'backup-dir new {rel}') + saved = bak / rel + if not saved.is_file(): + test_fail(f"--backup-dir did not preserve deep path for {rel}: {saved}") + if saved.read_bytes() != old[rel]: + test_fail(f"--backup-dir copy of {rel} does not hold the old content") + +# --- --backup-dir captures a deletion under --delete ------------------------ +rels, old = seed() +# Add a deep file to the destination that is absent from the source. +extra = os.path.join('d1', 'd2', 'd3', 'goner') +(TODIR / extra).write_bytes(b'about to be deleted\n') +run_rsync('-a', '--delete', '-b', f'--backup-dir={bak}', '--no-whole-file', + f'{src}/', f'{TODIR}/') +assert_not_exists(TODIR / extra, label='deleted file removed from dest') +saved = bak / extra +if not saved.is_file(): + test_fail(f"--backup-dir did not capture the deletion of {extra}") +if saved.read_bytes() != b'about to be deleted\n': + test_fail("captured deletion has the wrong content") + +print("backup-deep: suffix / backup-dir / delete-capture verified at depth") diff --git a/testsuite/delay-updates-deep_test.py b/testsuite/delay-updates-deep_test.py new file mode 100644 index 000000000..bd359fcd7 --- /dev/null +++ b/testsuite/delay-updates-deep_test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Property-level coverage of --delay-updates at depth. + +--delay-updates writes each updated file into a per-directory staging dir +(.~tmp~) during the transfer and only renames them into place at the very end, +so an interrupted run never leaves a half-written file visible. The staging dir +sits inside each destination directory, so the staging write and the +end-of-run rename are parent-directory operations the resolver restructure +rewrites; the ported delay-updates_test.py only exercises the tree root, so +this companion drives a >=3-deep tree. + +Asserts, at every level of the tree: + * a --delay-updates transfer reproduces the source and leaves no .~tmp~ + staging directory behind; + * a stale file pre-planted in a deep .~tmp~ staging dir is overwritten + cleanly and the staging dir is removed. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, make_tree, rmtree, run_rsync, test_fail, walk_dirs, + walk_files, +) + +src = FROMDIR +deepdir = os.path.join('d1', 'd2', 'd3') + + +def no_staging_left(): + leftover = [p for p in walk_dirs(TODIR) if p.name == '.~tmp~'] + if leftover: + test_fail(f"--delay-updates left staging dirs behind: {leftover}") + + +# --- initial --delay-updates over a deep tree ------------------------------- +rmtree(src) +rmtree(TODIR) +make_tree(src, depth=3, data=True, data_size=4096) +rels = [p.relative_to(src) for p in walk_files(src)] + +run_rsync('-a', '--delay-updates', f'{src}/', f'{TODIR}/') +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'delay-updates initial {rel}') +no_staging_left() + +# --- update every file, with a stale staging file planted deep -------------- +for rel in rels: + with open(src / rel, 'ab') as f: + f.write(b'\nupdated content\n') + +stage = TODIR / deepdir / '.~tmp~' +stage.mkdir(parents=True, exist_ok=True) +(stage / 'f3').write_bytes(b'stale staged junk\n') # must be overwritten + +run_rsync('-a', '--delay-updates', f'{src}/', f'{TODIR}/') +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'delay-updates update {rel}') +no_staging_left() + +print("delay-updates-deep: staging + clean overwrite verified at depth") diff --git a/testsuite/inplace_test.py b/testsuite/inplace_test.py new file mode 100644 index 000000000..e0c207929 --- /dev/null +++ b/testsuite/inplace_test.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Coverage of --inplace at depth. + +--inplace updates the destination file directly instead of writing a temp copy +and renaming it over the original, so across a delta update the destination +keeps the SAME inode. Without --inplace the receiver creates a fresh temp file +and renames it, giving the destination a NEW inode. Both behaviours hinge on +how the receiver resolves the destination directory and (for the default mode) +performs the temp->final rename, which the path restructure rewrites; verify +them on a file >=3 levels deep. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, make_tree, rmtree, run_rsync, test_fail, +) + +src = FROMDIR +deep = os.path.join('d1', 'd2', 'd3', 'f3') + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3, data=True, data_size=200000) + + +def inode(path): + return os.stat(path).st_ino + + +def modify_deep(): + # Rewrite a chunk in the middle so it is a genuine delta, not just a tail + # append. Bump mtime by a clear margin (the whole test runs inside one + # second, so a "now" touch would collide with the destination's mtime and + # the size-unchanged file would be skipped by the quick check). + p = src / deep + data = bytearray(p.read_bytes()) + data[1000:1100] = bytes((b ^ 0xFF) for b in data[1000:1100]) + p.write_bytes(bytes(data)) + st = os.stat(p) + os.utime(p, (st.st_atime, st.st_mtime + 100)) + + +# --- --inplace keeps the destination inode across a delta update ------------ +seed() +run_rsync('-a', f'{src}/', f'{TODIR}/') +ino_before = inode(TODIR / deep) + +modify_deep() +run_rsync('-a', '--inplace', '--no-whole-file', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='inplace content') +if inode(TODIR / deep) != ino_before: + test_fail("--inplace changed the destination inode at depth " + f"({ino_before} -> {inode(TODIR / deep)})") + +# --- control: the default (temp+rename) path replaces the inode ------------- +seed() +run_rsync('-a', f'{src}/', f'{TODIR}/') +ino_before = inode(TODIR / deep) + +modify_deep() +run_rsync('-a', '--no-whole-file', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='default content') +if inode(TODIR / deep) == ino_before: + test_fail("default (non-inplace) delta update unexpectedly kept the " + "destination inode at depth -- temp+rename did not run") + +print("inplace: same-inode update at depth verified; default replaces inode") diff --git a/testsuite/partial_test.py b/testsuite/partial_test.py new file mode 100644 index 000000000..97b6f69af --- /dev/null +++ b/testsuite/partial_test.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Coverage of --partial / --partial-dir at depth and across directory +boundaries. + +--partial keeps a partially transferred file so a later run can resume it. +--partial-dir=DIR keeps the partial in DIR instead of the destination file: +a RELATIVE dir is created in (and removed from) each destination file's own +directory; an ABSOLUTE dir is a reserved location that holds partials by +basename. All of this is parent- and cross-directory path resolution -- what +the resolver restructure rewrites -- so exercise it on a file several levels +deep, with the absolute partial-dir kept OUTSIDE the destination tree. + +Note: a *delta* resume from an absolute partial-dir currently fails whole-file +verification on master (it re-puts the partial and never converges). This test +therefore only asserts the cross-directory WRITE of the partial for that case +and completes it with --whole-file, which is the clearly-correct baseline. +""" + +import os +import signal +import subprocess +import time + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + assert_same, make_data_file, makepath, rmtree, rsync_argv, run_rsync, + test_fail, +) + +src = FROMDIR +deepdir = os.path.join('d1', 'd2', 'd3') +deep = os.path.join(deepdir, 'f3') +FULL = 12_000_000 + + +def seed_big(): + rmtree(src) + rmtree(TODIR) + makepath(src / deepdir) + make_data_file(src / deep, FULL) + + +def is_prefix(partial) -> bool: + pb = partial.read_bytes() + return 0 < len(pb) < FULL and (src / deep).read_bytes()[:len(pb)] == pb + + +def interrupt_transfer(extra_args, partial_path): + """Start a deliberately slow transfer, SIGTERM it once the receiver's + in-progress temp (.f3.XXXXXX) has some data, and wait for `partial_path` + (where this mode keeps the partial) to materialise. + + The bandwidth limit is low so the multi-second transfer cannot finish + before we interrupt it -- important under a loaded parallel run (-j16), + where the polling loop can lag by seconds. We then poll for the partial, + since rsync moves it into place from its exit_cleanup handler.""" + proc = subprocess.Popen( + rsync_argv('-a', '--no-whole-file', '--bwlimit=400', *extra_args, + f'{src}/', f'{TODIR}/'), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + tdir = TODIR / deepdir + caught = False + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if proc.poll() is not None: + break # exited before we caught it + if tdir.is_dir(): + temps = [p for p in tdir.glob('.f3.*') + if p.is_file() and p.stat().st_size > 0] + if temps: + caught = True + break + time.sleep(0.02) + proc.send_signal(signal.SIGTERM) + proc.wait() + if not caught: + test_fail("never caught an in-progress temp (transfer finished too " + "fast to interrupt)") + # rsync moves the partial into place from exit_cleanup; give it a moment. + pdeadline = time.monotonic() + 5 + while time.monotonic() < pdeadline: + if partial_path.is_file() and partial_path.stat().st_size > 0: + return + time.sleep(0.05) + + +# --- 1. --partial (no dir): partial kept in the dest file itself, at depth -- +seed_big() +interrupt_transfer(['--partial'], TODIR / deep) +if not (TODIR / deep).is_file() or not is_prefix(TODIR / deep): + test_fail("--partial did not leave a valid partial in the dest file") +run_rsync('-a', '--partial', '--no-whole-file', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='--partial resume') + +# --- 2. relative --partial-dir at depth: deterministic clean pre-seed ------- +rmtree(src) +rmtree(TODIR) +makepath(src / deepdir, TODIR / deepdir / '.rsync-partial') +make_data_file(src / deep, 1_000_000) +full = (src / deep).read_bytes() +(TODIR / deepdir / '.rsync-partial' / 'f3').write_bytes(full[:400_000]) +run_rsync('-a', '--partial-dir=.rsync-partial', '--no-whole-file', + f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='rel partial-dir preseed') +if (TODIR / deepdir / '.rsync-partial').exists(): + test_fail("relative --partial-dir not removed after the partial was used") + +# --- 3. relative --partial-dir at depth: interrupt then resume ------------- +seed_big() +part = TODIR / deepdir / '.rsync-partial' / 'f3' +interrupt_transfer(['--partial-dir=.rsync-partial'], part) +if not part.is_file() or not is_prefix(part): + test_fail("relative --partial-dir did not keep a valid partial at depth") +run_rsync('-a', '--partial-dir=.rsync-partial', '--no-whole-file', + f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='rel partial-dir resume') + +# --- 4. absolute --partial-dir OUTSIDE the tree (cross-dir): interrupt ----- +ext = SCRATCHDIR / 'partials' # sibling of from/ and to/ -- outside both +rmtree(ext) +ext.mkdir() +seed_big() +interrupt_transfer([f'--partial-dir={ext}'], ext / 'f3') +if not (ext / 'f3').is_file() or not is_prefix(ext / 'f3'): + test_fail("absolute --partial-dir did not write the partial to the " + "outside-tree dir") +run_rsync('-a', f'--partial-dir={ext}', '--whole-file', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='abs partial-dir resume') + +print("partial: --partial + relative/absolute --partial-dir verified at depth") diff --git a/testsuite/temp-dir_test.py b/testsuite/temp-dir_test.py new file mode 100644 index 000000000..63c37ea63 --- /dev/null +++ b/testsuite/temp-dir_test.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Coverage of --temp-dir (-T) at depth and across directory boundaries. + +--temp-dir tells the receiver to create its scratch/temp copies in DIR rather +than in the destination directory, then rename the finished file into place. +That rename crosses from the temp directory to a destination directory several +levels deep -- exactly the two-directory operation the path-resolution +restructure rewrites -- so it must be guarded at depth with the temp dir kept +OUTSIDE the destination tree. + +Asserts: + * a transfer with --temp-dir pointing outside the dest tree reproduces the + source byte-for-byte at every level; + * no temp/scratch files are left behind in the temp dir or the dest tree; + * a non-existent --temp-dir makes the receiver fail (so we know the option + is actually consulted, not silently ignored). +""" + +import os + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + assert_same, make_tree, rmtree, run_rsync, test_fail, walk_files, +) + +src = FROMDIR +tmp = SCRATCHDIR / 'scratch-temp' # sibling of from/ and to/ -- outside both +rmtree(src) +rmtree(TODIR) +rmtree(tmp) +tmp.mkdir() + +make_tree(src, depth=3, data=True) +rels = [p.relative_to(src) for p in walk_files(src)] + +# Transfer with the temp dir outside the destination tree. +run_rsync('-a', f'--temp-dir={tmp}', f'{src}/', f'{TODIR}/') + +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'temp-dir {rel}') + +# The temp dir must be clean afterwards (every scratch file renamed away). +leftover = sorted(p for p in tmp.rglob('*')) +if leftover: + test_fail(f"--temp-dir left scratch files behind: {leftover}") + +# No stray rsync temp files (.name.XXXXXX) anywhere in the dest tree. +strays = [p for p in TODIR.rglob('.*') if p.is_file()] +if strays: + test_fail(f"dest tree contains stray temp files: {strays}") + +# Negative: a missing temp dir must cause a receiver failure, proving the +# option is honoured rather than ignored. +rmtree(TODIR) +proc = run_rsync('-a', f'--temp-dir={SCRATCHDIR}/does-not-exist', + f'{src}/', f'{TODIR}/', check=False) +if proc.returncode == 0: + test_fail("--temp-dir pointing at a missing directory unexpectedly " + "succeeded") + +print("temp-dir: cross-dir rename at depth verified; missing temp dir fails") From d83097731c6b7752f74f199893a7a228d2c43f55 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 07:51:19 +1000 Subject: [PATCH 03/15] testsuite: structure / recursion / link coverage at depth Cover the structure and link options at >=3 levels and across directories, asserting each option's specific effect: links -l keeps a symlink, -L dereferences it, -k follows a directory symlink -- all on a symlink several levels deep. dirs -d copies the top layer (file + empty dir) without recursing. prune-empty-dirs -m drops empty chains and chains emptied by an exclude, keeps populated ones. hardlinks-deep -H preserves a hard link whose names live in different directories at depth; without -H they become separate inodes. delete-deep --delete removes a deep extraneous file/subtree; the four delete-timing variants agree; --max-delete caps deletions; --existing / --ignore-existing select/skip correctly. relative-implied -R mirrors an implied directory's mode at depth; --no-implied-dirs does not (proto 30+). Green on master and under --protocol=29/30 (the --no-implied-dirs sub-case is gated to protocol >= 30, where multi-component sender paths are accepted). Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/delete-deep_test.py | 85 ++++++++++++++++++++++++++++++ testsuite/dirs_test.py | 33 ++++++++++++ testsuite/hardlinks-deep_test.py | 36 +++++++++++++ testsuite/links_test.py | 65 +++++++++++++++++++++++ testsuite/prune-empty-dirs_test.py | 45 ++++++++++++++++ testsuite/relative-implied_test.py | 53 +++++++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 testsuite/delete-deep_test.py create mode 100644 testsuite/dirs_test.py create mode 100644 testsuite/hardlinks-deep_test.py create mode 100644 testsuite/links_test.py create mode 100644 testsuite/prune-empty-dirs_test.py create mode 100644 testsuite/relative-implied_test.py diff --git a/testsuite/delete-deep_test.py b/testsuite/delete-deep_test.py new file mode 100644 index 000000000..3525c2e3b --- /dev/null +++ b/testsuite/delete-deep_test.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Coverage of the --delete family at depth, plus --max-delete, --existing and +--ignore-existing. + +delete_test.py covers --del dry-run output, --remove-source-files and per-dir +protect filters; this companion asserts the concrete outcome of deletion deep +in the tree (the subtree walk the resolver restructure touches) and the +controls that bound or invert it. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_not_exists, assert_same, make_tree, makepath, rmtree, run_rsync, + test_fail, walk_files, +) + +src = FROMDIR + + +def seed_src(): + rmtree(src) + make_tree(src, depth=3) + return [p.relative_to(src) for p in walk_files(src)] + + +def fresh_dest(): + rmtree(TODIR) + run_rsync('-a', f'{src}/', f'{TODIR}/') + + +# --- --delete removes a deep extraneous file and subtree -------------------- +rels = seed_src() +fresh_dest() +makepath(TODIR / 'd1' / 'd2' / 'extra') +(TODIR / 'd1' / 'd2' / 'extra' / 'junk').write_text("x\n") +(TODIR / 'd1' / 'd2' / 'orphan').write_text("y\n") +run_rsync('-a', '--delete', f'{src}/', f'{TODIR}/') +assert_not_exists(TODIR / 'd1' / 'd2' / 'extra', label='--delete deep dir') +assert_not_exists(TODIR / 'd1' / 'd2' / 'orphan', label='--delete deep file') +for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'--delete kept {rel}') + +# --- every delete-timing variant yields the same deep deletion -------------- +for variant in ('--delete-before', '--delete-during', '--delete-delay', + '--delete-after'): + fresh_dest() + (TODIR / 'd1' / 'd2' / 'gone.txt').write_text("z\n") + run_rsync('-a', variant, f'{src}/', f'{TODIR}/') + assert_not_exists(TODIR / 'd1' / 'd2' / 'gone.txt', label=f'{variant} deep') + +# --- --max-delete caps the number of deletions ------------------------------ +fresh_dest() +for i in range(5): + (TODIR / f'extra{i}').write_text("e\n") +run_rsync('-a', '--delete', '--max-delete=2', f'{src}/', f'{TODIR}/', + check=False) # rsync exits 25 when the limit is hit +remaining = list(TODIR.glob('extra*')) +if len(remaining) != 3: + test_fail(f"--max-delete=2 should leave 3 of 5 extras, found " + f"{len(remaining)}: {remaining}") + +# --- --existing only updates files already present (creates nothing) --------- +seed_src() +rmtree(TODIR) +makepath(TODIR / 'd1') +(TODIR / 'd1' / 'f1').write_text("old\n") +run_rsync('-a', '--existing', f'{src}/', f'{TODIR}/') +assert_same(TODIR / 'd1' / 'f1', src / 'd1' / 'f1', + label='--existing updated existing deep file') +assert_not_exists(TODIR / 'f0', label='--existing did not create new top file') +assert_not_exists(TODIR / 'd1' / 'd2', label='--existing did not create new dir') + +# --- --ignore-existing skips present files, creates the rest ----------------- +seed_src() +rmtree(TODIR) +makepath(TODIR / 'd1') +(TODIR / 'd1' / 'f1').write_text("KEEP THIS\n") +run_rsync('-a', '--ignore-existing', f'{src}/', f'{TODIR}/') +if (TODIR / 'd1' / 'f1').read_text() != "KEEP THIS\n": + test_fail("--ignore-existing overwrote an existing deep file") +assert_same(TODIR / 'f0', src / 'f0', label='--ignore-existing created new file') + +print("delete-deep: delete family, max-delete, existing/ignore-existing at depth") diff --git a/testsuite/dirs_test.py b/testsuite/dirs_test.py new file mode 100644 index 000000000..3772734a3 --- /dev/null +++ b/testsuite/dirs_test.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Coverage of -d / --dirs: transfer directories without recursing into them. + +-d copies the entries directly named (or directly inside a trailing-slash +source): regular files at the top level are copied, and a top-level directory +is created as an empty directory -- its contents are NOT transferred. Verify +that on a tree that is several levels deep, only the top layer materialises. +""" + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, make_tree, rmtree, run_rsync, test_fail, +) + +src = FROMDIR +rmtree(src) +rmtree(TODIR) +make_tree(src, depth=3) # f0 at root; d1/{f1, d2/{f2, d3/f3}} + +run_rsync('-d', f'{src}/', f'{TODIR}/') + +# The top-level file is copied. +assert_same(TODIR / 'f0', src / 'f0', label='-d top-level file') +# The top-level directory is created... +if not (TODIR / 'd1').is_dir(): + test_fail("-d did not create the top-level directory") +# ...but NOT recursed into. +if (TODIR / 'd1' / 'f1').exists(): + test_fail("-d recursed into a directory (f1 should not exist)") +if list((TODIR / 'd1').iterdir()): + test_fail("-d populated the directory; it should be empty") + +print("dirs: -d copies the top layer without recursing") diff --git a/testsuite/hardlinks-deep_test.py b/testsuite/hardlinks-deep_test.py new file mode 100644 index 000000000..def0aa26b --- /dev/null +++ b/testsuite/hardlinks-deep_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Coverage of -H across directory boundaries. + +hardlinks_test.py exercises -H on sibling files at the tree root; this +companion checks that -H preserves a hard link whose two names live in +DIFFERENT directories several levels deep (the cross-directory case the +resolver restructure touches), and that without -H the names become +independent inodes. +""" + +from rsyncfns import ( + FROMDIR, TODIR, + assert_hardlinked, assert_not_hardlinked, makepath, rmtree, run_rsync, +) +import os + +src = FROMDIR +a = os.path.join('a', 'aa', 'orig') +b = os.path.join('b', 'bb', 'hardlink') + +rmtree(src) +rmtree(TODIR) +makepath(src / 'a' / 'aa', src / 'b' / 'bb') +(src / a).write_text("shared content across directories\n") +os.link(src / a, src / b) # one inode, two names in different dirs + +# -H preserves the cross-directory hard link. +run_rsync('-aH', f'{src}/', f'{TODIR}/') +assert_hardlinked(TODIR / a, TODIR / b, label='-H cross-dir hardlink') + +# Without -H the two names are copied as independent files. +rmtree(TODIR) +run_rsync('-a', f'{src}/', f'{TODIR}/') +assert_not_hardlinked(TODIR / a, TODIR / b, label='no -H => separate inodes') + +print("hardlinks-deep: -H preserves a cross-directory hard link at depth") diff --git a/testsuite/links_test.py b/testsuite/links_test.py new file mode 100644 index 000000000..f7b412def --- /dev/null +++ b/testsuite/links_test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Coverage of symlink-handling options -l / -L / -k at depth. + + -l (--links) copy a symlink as a symlink (target preserved); + -L (--copy-links) replace a symlink with the file/dir it points to; + -k (--copy-dirlinks) treat a symlink-to-directory as the real directory. + +(-K --keep-dirlinks -- the receiver-side follow of an in-tree directory +symlink, issue #715 -- is covered by symlink-dirlink-basis_test.py and needs +the secure resolver; here we cover the portable source-side options, all on a +symlink that lives several directory levels deep.) +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_is_symlink, assert_same, make_tree, rmtree, run_rsync, test_fail, +) + +src = FROMDIR +deepdir = os.path.join('d1', 'd2', 'd3') + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3) + os.symlink('f3', src / deepdir / 'sl') # file symlink, deep + (src / deepdir / 'realdir').mkdir() + (src / deepdir / 'realdir' / 'inside').write_text("inside\n") + os.symlink('realdir', src / deepdir / 'dirlink') # dir symlink, deep + + +# --- -l: symlinks preserved as symlinks at depth ---------------------------- +seed() +run_rsync('-rl', f'{src}/', f'{TODIR}/') +assert_is_symlink(TODIR / deepdir / 'sl', target='f3', label='-l file symlink') +assert_is_symlink(TODIR / deepdir / 'dirlink', target='realdir', + label='-l dir symlink') + +# --- -L: symlinks dereferenced into their referents at depth ----------------- +seed() +run_rsync('-rL', f'{src}/', f'{TODIR}/') +if os.path.islink(TODIR / deepdir / 'sl'): + test_fail("-L left a file symlink at depth instead of dereferencing it") +assert_same(TODIR / deepdir / 'sl', src / deepdir / 'f3', label='-L deref file') +if os.path.islink(TODIR / deepdir / 'dirlink'): + test_fail("-L left a dir symlink at depth instead of dereferencing it") +assert_same(TODIR / deepdir / 'dirlink' / 'inside', + src / deepdir / 'realdir' / 'inside', label='-L deref dir') + +# --- -k: only the dir-symlink is followed; the file symlink stays a symlink -- +seed() +run_rsync('-rlk', f'{src}/', f'{TODIR}/') +if os.path.islink(TODIR / deepdir / 'dirlink'): + test_fail("-k left the dir symlink as a symlink") +if not (TODIR / deepdir / 'dirlink').is_dir(): + test_fail("-k did not turn the dir symlink into a real directory") +assert_same(TODIR / deepdir / 'dirlink' / 'inside', + src / deepdir / 'realdir' / 'inside', label='-k dir contents') +assert_is_symlink(TODIR / deepdir / 'sl', target='f3', + label='-k keeps the file symlink') + +print("links: -l preserves, -L dereferences, -k follows dir-symlinks (at depth)") diff --git a/testsuite/prune-empty-dirs_test.py b/testsuite/prune-empty-dirs_test.py new file mode 100644 index 000000000..c53adfbd1 --- /dev/null +++ b/testsuite/prune-empty-dirs_test.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Coverage of -m / --prune-empty-dirs at depth. + +--prune-empty-dirs drops directory chains that would end up empty at the +destination -- both chains that are empty in the source and chains that become +empty because a filter excluded their only files. Populated chains are kept. +""" + +from rsyncfns import ( + FROMDIR, TODIR, + assert_not_exists, assert_same, makepath, rmtree, run_rsync, +) + +src = FROMDIR + + +def reseed(): + rmtree(src) + rmtree(TODIR) + + +# --- a deep empty chain is pruned; a deep populated chain is kept ------------ +reseed() +makepath(src / 'empty' / 'e1' / 'e2', src / 'full' / 'd1' / 'd2') +(src / 'full' / 'd1' / 'd2' / 'file').write_text("data\n") + +run_rsync('-a', '-m', f'{src}/', f'{TODIR}/') +assert_not_exists(TODIR / 'empty', label='-m pruned an empty chain') +assert_same(TODIR / 'full' / 'd1' / 'd2' / 'file', + src / 'full' / 'd1' / 'd2' / 'file', label='-m kept populated chain') + +# --- a chain emptied by an exclude filter is also pruned -------------------- +reseed() +makepath(src / 'mixed' / 'sub', src / 'onlylogs' / 'sub') +(src / 'mixed' / 'sub' / 'keep.txt').write_text("k\n") +(src / 'mixed' / 'sub' / 'drop.log').write_text("d\n") +(src / 'onlylogs' / 'sub' / 'a.log').write_text("a\n") + +run_rsync('-a', '-m', '--exclude=*.log', f'{src}/', f'{TODIR}/') +assert_same(TODIR / 'mixed' / 'sub' / 'keep.txt', + src / 'mixed' / 'sub' / 'keep.txt', label='-m kept non-empty dir') +assert_not_exists(TODIR / 'onlylogs', + label='-m pruned a dir emptied by an exclude') + +print("prune-empty-dirs: empty and filter-emptied chains pruned, populated kept") diff --git a/testsuite/relative-implied_test.py b/testsuite/relative-implied_test.py new file mode 100644 index 000000000..167d79dfe --- /dev/null +++ b/testsuite/relative-implied_test.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Coverage of -R implied directories and --no-implied-dirs at depth. + +With -R the directories implied by a source path are recreated at the +destination AND have their attributes mirrored from the source. With +--no-implied-dirs those implied directories are still created as needed but get +default attributes instead of the source's. Verify the distinction on an +implied directory carrying a non-default mode, several levels deep. +""" + +import os +import stat + +from rsyncfns import ( + SCRATCHDIR, TODIR, + assert_mode, assert_same, forced_protocol, makepath, rmtree, run_rsync, + test_fail, +) + +base = SCRATCHDIR / 'rbase' +rmtree(base) +rmtree(TODIR) +makepath(base / 'a' / 'b' / 'c') +os.chmod(base / 'a' / 'b', 0o750) # distinctive mode on an implied dir +(base / 'a' / 'b' / 'c' / 'file').write_text("data\n") + +os.chdir(base / 'a') + +# -R: implied dirs b and c are recreated with the source's attributes. +run_rsync('-aR', 'b/c/file', f'{TODIR}/') +assert_mode(TODIR / 'b', 0o750, label='-R mirrors implied-dir mode') +assert_same(TODIR / 'b' / 'c' / 'file', base / 'a' / 'b' / 'c' / 'file', + label='-R deep file') + +# --no-implied-dirs: implied dir b is created with default (not source) attrs. +# At protocol 29 the generator rejects a multi-component path that has no +# transmitted directory entries ("invalid path from sender"), so this half is +# protocol-30+. +proto = forced_protocol() +if proto is not None and proto < 30: + print(f"relative-implied: protocol {proto} -- skipping --no-implied-dirs " + "(the multi-component path is rejected by the proto-29 generator)") +else: + rmtree(TODIR) + run_rsync('-aR', '--no-implied-dirs', 'b/c/file', f'{TODIR}/') + m = stat.S_IMODE(os.stat(TODIR / 'b').st_mode) + if m == 0o750: + test_fail("--no-implied-dirs unexpectedly mirrored the source mode " + "0750 onto the implied directory") + assert_same(TODIR / 'b' / 'c' / 'file', base / 'a' / 'b' / 'c' / 'file', + label='--no-implied-dirs deep file') + +print("relative-implied: -R mirrors implied-dir attrs; --no-implied-dirs does not") From 7ebb20765805df2c6866d99a67e38ccb4ab34006 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 07:54:28 +1000 Subject: [PATCH 04/15] testsuite: metadata preservation coverage at depth Set each attribute distinctively on a file AND a directory at every level of a >=3-deep tree and verify it per entry after transfer (metadata is applied as a single-component op on an entry whose parent chain the resolver restructure rewrites): metadata-depth -p preserves exact file/dir modes; -t preserves file mtimes; --chmod=D710,F600 rewrites them. omit-times -O omits directory times (files still preserved); -J omits symlink times. sparse -S preserves a deep file's hole (allocated << size); --no-sparse fills it. xattrs-depth -X reproduces a user xattr on every entry (gated on xattr support). acls-depth -A reproduces a POSIX ACL on every entry (gated on ACL support + setfacl/getfacl). ownership-depth --groupmap and --chown=:GROUP remap the group of every entry (non-root, to a secondary group); -o/--usermap gated on root. All green on master and under --protocol=29/30. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/acls-depth_test.py | 63 ++++++++++++++++++++++++ testsuite/metadata-depth_test.py | 57 +++++++++++++++++++++ testsuite/omit-times_test.py | 58 ++++++++++++++++++++++ testsuite/ownership-depth_test.py | 82 +++++++++++++++++++++++++++++++ testsuite/sparse_test.py | 59 ++++++++++++++++++++++ testsuite/xattrs-depth_test.py | 52 ++++++++++++++++++++ 6 files changed, 371 insertions(+) create mode 100644 testsuite/acls-depth_test.py create mode 100644 testsuite/metadata-depth_test.py create mode 100644 testsuite/omit-times_test.py create mode 100644 testsuite/ownership-depth_test.py create mode 100644 testsuite/sparse_test.py create mode 100644 testsuite/xattrs-depth_test.py diff --git a/testsuite/acls-depth_test.py b/testsuite/acls-depth_test.py new file mode 100644 index 000000000..308be7650 --- /dev/null +++ b/testsuite/acls-depth_test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Coverage of -A (acls) at depth. + +acls_test.py exercises a shallow tree; this companion sets a distinctive POSIX +ACL on a file AND a directory at every level of a >=3-deep tree and checks that +-A reproduces them (the ACL is applied per entry, on a name whose parent chain +the resolver restructure rewrites). +""" + +import os +import shutil +import subprocess + +from rsyncfns import ( + FROMDIR, TODIR, + make_tree, rmtree, run_rsync, test_fail, test_skipped, + walk_dirs, walk_files, +) + +vv = run_rsync('-VV', check=True, capture_output=True).stdout +if '"ACLs": true' not in vv: + test_skipped("rsync built without ACL support") +if not (shutil.which('setfacl') and shutil.which('getfacl')): + test_skipped("setfacl/getfacl not available") + +src = FROMDIR +rmtree(src) +rmtree(TODIR) +make_tree(src, depth=3) + +entries = [p.relative_to(src) for p in (walk_dirs(src) + walk_files(src))] +entries.sort() + +# Grant uid 0 an explicit r-x ACL on every entry (valid for a non-root owner to +# set on its own files; needs no extra accounts). +for rel in entries: + r = subprocess.run(['setfacl', '-m', 'u:0:r-x', str(src / rel)]) + if r.returncode != 0: + test_skipped("filesystem does not support setting ACLs") + + +def getfacl(path): + # Strip the comment header (# file:/# owner:/# group:) so the comparison + # is path-independent; this getfacl doesn't accept the GNU -c flag. + out = subprocess.check_output(['getfacl', str(path)], text=True, + stderr=subprocess.DEVNULL) + return ''.join(ln for ln in out.splitlines(keepends=True) + if not ln.startswith('#')) + + +run_rsync('-aA', f'{src}/', f'{TODIR}/') + +for rel in entries: + want = getfacl(src / rel) + got = getfacl(TODIR / rel) + # getfacl renders uid 0 as "root"; accept either spelling. + if 'user:root:r-x' not in got and 'user:0:r-x' not in got: + test_fail(f"-A did not reproduce the named-user ACL on {rel}:\n{got}") + if want != got: + test_fail(f"-A: ACL of {rel} differs\n--- source ---\n{want}" + f"--- dest ---\n{got}") + +print("acls-depth: -A reproduced a POSIX ACL on every entry at depth") diff --git a/testsuite/metadata-depth_test.py b/testsuite/metadata-depth_test.py new file mode 100644 index 000000000..f8ba7a077 --- /dev/null +++ b/testsuite/metadata-depth_test.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Coverage of -p (perms), -t (times) and --chmod at depth. + +Each attribute is set distinctively on a file AND a directory at every level of +a >=3-deep tree, then checked per entry after the transfer -- the metadata is +applied as a single-component operation on an entry whose parent chain the +resolver restructure rewrites, so it must be verified deep, not just at the +root. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_mode, assert_mtime_close, make_tree, rmtree, run_rsync, test_fail, + walk_dirs, walk_files, +) + +src = FROMDIR +FILE_MODE = 0o640 +DIR_MODE = 0o750 +BASE_MTIME = 1_400_000_000 # a fixed, clearly-old timestamp + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3) + for i, f in enumerate(walk_files(src)): + os.chmod(f, FILE_MODE) + os.utime(f, (BASE_MTIME + i * 100, BASE_MTIME + i * 100)) + for d in walk_dirs(src): + os.chmod(d, DIR_MODE) + + +# --- -p preserves exact file and directory modes at every level ------------- +seed() +run_rsync('-rlpt', f'{src}/', f'{TODIR}/') +for f in walk_files(src): + assert_mode(TODIR / f.relative_to(src), FILE_MODE, label=f'-p file {f.name}') +for d in walk_dirs(src): + assert_mode(TODIR / d.relative_to(src), DIR_MODE, label=f'-p dir {d.name}') + +# --- -t preserves file mtimes at every level -------------------------------- +for f in walk_files(src): + rel = f.relative_to(src) + assert_mtime_close(TODIR / rel, f.stat().st_mtime, label=f'-t {rel}') + +# --- --chmod rewrites modes at every level ---------------------------------- +seed() +run_rsync('-a', '--chmod=D710,F600', f'{src}/', f'{TODIR}/') +for f in walk_files(src): + assert_mode(TODIR / f.relative_to(src), 0o600, label=f'--chmod file {f.name}') +for d in walk_dirs(src): + assert_mode(TODIR / d.relative_to(src), 0o710, label=f'--chmod dir {d.name}') + +print("metadata-depth: -p / -t / --chmod verified per entry at depth") diff --git a/testsuite/omit-times_test.py b/testsuite/omit-times_test.py new file mode 100644 index 000000000..a532c85f9 --- /dev/null +++ b/testsuite/omit-times_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Coverage of -O (--omit-dir-times) and -J (--omit-link-times) at depth. + +-O preserves file mtimes but leaves directory mtimes alone; -J does the same +for symlinks. Verify the distinction deep in the tree: the preserved entries +match the source, the omitted ones do not. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_mtime_close, make_tree, rmtree, run_rsync, test_fail, + walk_dirs, walk_files, +) + +src = FROMDIR +OLD = 1_400_000_000 + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3) + for p in list(walk_files(src)) + list(walk_dirs(src)): + os.utime(p, (OLD, OLD)) + + +# --- -O: file mtimes preserved, directory mtimes omitted -------------------- +seed() +run_rsync('-rlt', '-O', f'{src}/', f'{TODIR}/') +for f in walk_files(src): + assert_mtime_close(TODIR / f.relative_to(src), OLD, label=f'-O file {f.name}') +omitted = False +for d in walk_dirs(src): + if abs(os.stat(TODIR / d.relative_to(src)).st_mtime - OLD) > 1: + omitted = True # at least one dir keeps a fresh (now) mtime +if not omitted: + test_fail("-O did not omit directory times -- every dir mtime matched the " + "source") + +# --- -J: symlink mtime omitted (where the platform records symlink mtimes) -- +seed() +deep = os.path.join('d1', 'd2', 'd3') +os.symlink('f3', src / deep / 'sl') +try: + os.utime(src / deep / 'sl', (OLD, OLD), follow_symlinks=False) +except (NotImplementedError, OSError): + print("omit-times: -J check skipped (no symlink-mtime support here)") +else: + run_rsync('-rlt', '-J', f'{src}/', f'{TODIR}/') + dst = TODIR / deep / 'sl' + if not os.path.islink(dst): + test_fail("-J test: symlink was not copied") + if abs(os.lstat(dst).st_mtime - OLD) <= 1: + test_fail("-J did not omit the symlink mtime") + +print("omit-times: -O omits dir times, -J omits link times (at depth)") diff --git a/testsuite/ownership-depth_test.py b/testsuite/ownership-depth_test.py new file mode 100644 index 000000000..6facae257 --- /dev/null +++ b/testsuite/ownership-depth_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Coverage of --groupmap / --chown / --usermap / -o at depth. + +Remapping is applied to a file AND a directory at every level. As root we can +remap to an arbitrary uid/gid (root may always chown/chgrp), so the uid side is +covered too. As a normal user we can still remap the group to a secondary group +we belong to; the uid side then needs root and is skipped. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + get_rootuid, get_testuid, make_tree, rmtree, rsync_getgroups, run_rsync, + test_fail, test_skipped, walk_dirs, walk_files, +) + +src = FROMDIR +is_root = get_testuid() == get_rootuid() +prim = os.getgid() + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3) + entries = [p.relative_to(src) for p in (walk_dirs(src) + walk_files(src))] + # Normalise the source group so a prim->target remap is observable. + for rel in entries: + if os.stat(src / rel).st_gid != prim: + os.chown(src / rel, -1, prim) + return entries + + +def assert_all(entries, *, gid=None, uid=None, label=''): + for rel in entries: + st = os.stat(TODIR / rel) + if gid is not None and st.st_gid != gid: + test_fail(f"{label}: group of {rel} is {st.st_gid}, expected {gid}") + if uid is not None and st.st_uid != uid: + test_fail(f"{label}: owner of {rel} is {st.st_uid}, expected {uid}") + + +if is_root: + # Root may assign any numeric id (it need not exist); pick targets that + # differ from the source's ids so the remap is observable. + target_gid = 1 if prim == 0 else 0 + target_uid = 1 if get_testuid() == 0 else 0 + + entries = seed() + run_rsync('-a', f'--groupmap={prim}:{target_gid}', f'{src}/', f'{TODIR}/') + assert_all(entries, gid=target_gid, label='--groupmap (root)') + + entries = seed() + run_rsync('-a', f'--chown=:{target_gid}', f'{src}/', f'{TODIR}/') + assert_all(entries, gid=target_gid, label='--chown group (root)') + + entries = seed() + run_rsync('-a', f'--usermap=*:{target_uid}', f'{src}/', f'{TODIR}/') + assert_all(entries, uid=target_uid, label='--usermap (root)') + + entries = seed() + run_rsync('-a', f'--chown={target_uid}:{target_gid}', f'{src}/', f'{TODIR}/') + assert_all(entries, uid=target_uid, gid=target_gid, + label='--chown user:group (root)') + print("ownership-depth: --groupmap/--chown/--usermap verified at depth (root)") +else: + groups = [int(g) for g in rsync_getgroups()] + secs = [g for g in groups if g != prim] + if not secs: + test_skipped("non-root with no secondary group to remap to") + sec = secs[0] + + entries = seed() + run_rsync('-a', f'--groupmap={prim}:{sec}', f'{src}/', f'{TODIR}/') + assert_all(entries, gid=sec, label='--groupmap') + + entries = seed() + run_rsync('-a', f'--chown=:{sec}', f'{src}/', f'{TODIR}/') + assert_all(entries, gid=sec, label='--chown group') + print("ownership-depth: --groupmap/--chown group remap verified at depth " + "(-o/--usermap user remap needs root -- skipped)") diff --git a/testsuite/sparse_test.py b/testsuite/sparse_test.py new file mode 100644 index 000000000..898ed8c96 --- /dev/null +++ b/testsuite/sparse_test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Coverage of -S (--sparse) at depth. + +A file with a large hole, several directory levels deep, should arrive sparse +(its allocated blocks much smaller than its apparent size) and byte-identical +when copied with -S. + +We do NOT assert the converse (that a plain copy fills the hole): whether a +zero-run is stored as a hole when -S is absent is the filesystem's choice, not +rsync's -- ZFS/APFS transparently sparsify zero blocks, so a --no-sparse copy +can legitimately stay sparse there. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, makepath, rmtree, run_rsync, test_fail, test_skipped, +) + +src = FROMDIR +deep = os.path.join('d1', 'd2', 'd3', 'holey') +SIZE = 4 * 1024 * 1024 + + +def make_sparse(path): + with open(path, 'wb') as f: + f.write(b'head') + f.seek(SIZE - 4) + f.write(b'tail') + + +def allocated(path): + return os.stat(path).st_blocks * 512 + + +rmtree(src) +rmtree(TODIR) +makepath(src / 'd1' / 'd2' / 'd3') +make_sparse(src / deep) + +# Confirm the source filesystem actually made a sparse file; otherwise the +# whole premise (and any dest comparison) is meaningless here. +if allocated(src / deep) >= SIZE: + test_skipped("source filesystem did not create a sparse file") + +# --- with -S the hole is preserved at the destination ----------------------- +run_rsync('-a', '-S', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='-S content') +if allocated(TODIR / deep) >= SIZE: + test_fail(f"-S did not preserve the hole at depth " + f"(allocated {allocated(TODIR / deep)} for a {SIZE}-byte file)") + +# --- a plain copy reproduces the content too (allocation is FS-defined) ------ +rmtree(TODIR) +run_rsync('-a', '--no-sparse', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='no-sparse content') + +print("sparse: -S preserves a deep hole; content correct with and without it") diff --git a/testsuite/xattrs-depth_test.py b/testsuite/xattrs-depth_test.py new file mode 100644 index 000000000..c090286c6 --- /dev/null +++ b/testsuite/xattrs-depth_test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Coverage of -X (xattrs) at depth. + +xattrs_test.py exercises a shallow tree; this companion sets a distinctive user +xattr on a file AND a directory at every level of a >=3-deep tree and checks +that -X reproduces them all (the xattr is applied per entry, on a name whose +parent chain the resolver restructure rewrites). +""" + +import os +import sys + +from rsyncfns import ( + FROMDIR, TODIR, + make_tree, rmtree, run_rsync, test_fail, test_skipped, + walk_dirs, walk_files, xattr_dump, xattr_set, xattrs_supported, +) + +if not xattrs_supported(): + test_skipped("rsync built without xattr support (or no xattr tooling here)") + +src = FROMDIR +rmtree(src) +rmtree(TODIR) +make_tree(src, depth=3) + +entries = [p.relative_to(src) for p in (walk_dirs(src) + walk_files(src))] +entries.sort() + +os.chdir(src) +try: + for i, rel in enumerate(entries): + xattr_set('depth', f'value-{i}-{rel}', str(rel)) +except OSError as e: + test_skipped(f"unable to set an xattr on this filesystem: {e}") + +want = xattr_dump(*[str(r) for r in entries]) + +run_rsync('-aX', '-f-x_system.*', '-f-x_security.*', '--super', + f'{src}/', f'{TODIR}/') + +os.chdir(TODIR) +got = xattr_dump(*[str(r) for r in entries]) + +if got != want: + from difflib import unified_diff + sys.stdout.write(''.join(unified_diff( + want.splitlines(keepends=True), got.splitlines(keepends=True), + fromfile='source', tofile='dest'))) + test_fail("xattrs differ between source and destination at depth") + +print("xattrs-depth: -X reproduced a user xattr on every entry at depth") From 9012403075aa97a992eb25377af781a55b15cdff Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 07:55:45 +1000 Subject: [PATCH 05/15] testsuite: filtering coverage at depth Assert exactly which entries are/aren't transferred, deep in the tree: filter-depth --exclude/--include precedence on files at every level, and a -F per-directory .rsync-filter loaded from a deep dir that applies to that subtree only (not above it). cvs-exclude -C built-in cruft patterns (*.o, *~) at every level plus a deep per-directory .cvsignore scoped to its subtree. size-filter --max-size / --min-size select the right files all the way down. files-from-depth --files-from selects only the listed deep paths (implied parents created); --from0 NUL-delimited; --exclude-from / --include-from filter at depth. (--existing / --ignore-existing are covered in delete-deep_test.py.) Green on master and under --protocol=29/30. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/cvs-exclude_test.py | 46 +++++++++++++++++++ testsuite/files-from-depth_test.py | 68 ++++++++++++++++++++++++++++ testsuite/filter-depth_test.py | 71 ++++++++++++++++++++++++++++++ testsuite/size-filter_test.py | 50 +++++++++++++++++++++ 4 files changed, 235 insertions(+) create mode 100644 testsuite/cvs-exclude_test.py create mode 100644 testsuite/files-from-depth_test.py create mode 100644 testsuite/filter-depth_test.py create mode 100644 testsuite/size-filter_test.py diff --git a/testsuite/cvs-exclude_test.py b/testsuite/cvs-exclude_test.py new file mode 100644 index 000000000..3416d5ce0 --- /dev/null +++ b/testsuite/cvs-exclude_test.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Coverage of -C / --cvs-exclude at depth. + +-C ignores the usual CVS cruft (object files, core, editor backups, VCS dirs, +...) and also honours a per-directory .cvsignore. Verify both the built-in +patterns and a deep .cvsignore on a >=3-level tree. +""" + +from rsyncfns import ( + FROMDIR, TODIR, + assert_exists, assert_not_exists, makepath, rmtree, run_rsync, +) + +src = FROMDIR +rmtree(src) +rmtree(TODIR) +makepath(src / 'd1' / 'd2' / 'd3') + +# A real file plus default-CVS-cruft at every level. +cur = src +for lvl in range(4): + (cur / f'real{lvl}.c').write_text('code\n') + (cur / f'obj{lvl}.o').write_text('obj\n') # *.o is built-in cruft + (cur / f'back{lvl}~').write_text('backup\n') # *~ is built-in cruft + cur = cur / f'd{lvl + 1}' + +# A per-directory .cvsignore deep in the tree adds "*.junk" for that subtree. +(src / 'd1' / 'd2' / '.cvsignore').write_text('*.junk\n') +(src / 'd1' / 'd2' / 'local.junk').write_text('j\n') +(src / 'top.junk').write_text('j\n') # not covered by that .cvsignore + +run_rsync('-aC', f'{src}/', f'{TODIR}/') + +cur = TODIR +for lvl in range(4): + assert_exists(cur / f'real{lvl}.c', label=f'-C kept real L{lvl}') + assert_not_exists(cur / f'obj{lvl}.o', label=f'-C dropped *.o L{lvl}') + assert_not_exists(cur / f'back{lvl}~', label=f'-C dropped *~ L{lvl}') + cur = cur / f'd{lvl + 1}' + +# .cvsignore is scoped to its directory subtree. +assert_not_exists(TODIR / 'd1' / 'd2' / 'local.junk', + label='-C deep .cvsignore applied') +assert_exists(TODIR / 'top.junk', label='-C deep .cvsignore not applied above') + +print("cvs-exclude: built-in patterns + deep .cvsignore honoured at depth") diff --git a/testsuite/files-from-depth_test.py b/testsuite/files-from-depth_test.py new file mode 100644 index 000000000..beaa85752 --- /dev/null +++ b/testsuite/files-from-depth_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Coverage of --files-from, -0/--from0, --exclude-from, --include-from at depth. + +--files-from selects exactly the listed source-relative paths (creating their +implied parent dirs); --from0 makes the list NUL-delimited; --exclude-from / +--include-from read filter patterns from a file. All resolve names several +levels deep. +""" + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + assert_exists, assert_not_exists, assert_same, make_tree, rmtree, + run_rsync, +) + +src = FROMDIR +listed = ['d1/f1', 'd1/d2/d3/f3'] +unlisted = ['f0', 'd1/d2/f2'] + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3) + + +# --- --files-from selects only the listed deep paths ------------------------ +seed() +lf = SCRATCHDIR / 'files.lst' +lf.write_text('\n'.join(listed) + '\n') +run_rsync('-a', f'--files-from={lf}', f'{src}/', f'{TODIR}/') +for rel in listed: + assert_same(TODIR / rel, src / rel, label=f'--files-from {rel}') +for rel in unlisted: + assert_not_exists(TODIR / rel, label=f'--files-from excluded {rel}') + +# --- --from0: the same list, NUL-delimited ---------------------------------- +rmtree(TODIR) +lf0 = SCRATCHDIR / 'files0.lst' +lf0.write_bytes(b'\0'.join(p.encode() for p in listed) + b'\0') +run_rsync('-a', '--from0', f'--files-from={lf0}', f'{src}/', f'{TODIR}/') +for rel in listed: + assert_same(TODIR / rel, src / rel, label=f'--from0 {rel}') +for rel in unlisted: + assert_not_exists(TODIR / rel, label=f'--from0 excluded {rel}') + +# --- --exclude-from drops matching files at depth --------------------------- +seed() +(src / 'a.skip').write_text('s\n') +(src / 'd1' / 'd2' / 'a.skip').write_text('s\n') +ef = SCRATCHDIR / 'excl.lst' +ef.write_text('*.skip\n') +run_rsync('-a', f'--exclude-from={ef}', f'{src}/', f'{TODIR}/') +assert_not_exists(TODIR / 'a.skip', label='--exclude-from top') +assert_not_exists(TODIR / 'd1' / 'd2' / 'a.skip', label='--exclude-from deep') +assert_same(TODIR / 'd1' / 'd2' / 'f2', src / 'd1' / 'd2' / 'f2', + label='--exclude-from kept others') + +# --- --include-from keeps only matching files at depth ---------------------- +seed() +(src / 'd1' / 'd2' / 'k.keepme').write_text('k\n') +inc = SCRATCHDIR / 'inc.lst' +inc.write_text('*/\n*.keepme\n') +run_rsync('-a', f'--include-from={inc}', '--exclude=*', f'{src}/', f'{TODIR}/') +assert_exists(TODIR / 'd1' / 'd2' / 'k.keepme', label='--include-from kept') +assert_not_exists(TODIR / 'd1' / 'd2' / 'f2', label='--include-from excluded rest') + +print("files-from-depth: --files-from/--from0/--exclude-from/--include-from at depth") diff --git a/testsuite/filter-depth_test.py b/testsuite/filter-depth_test.py new file mode 100644 index 000000000..9e4c99f04 --- /dev/null +++ b/testsuite/filter-depth_test.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Coverage of --exclude / --include / --filter / -F at depth. + +The interesting case for the resolver restructure is a per-directory merge file +(-F reads .rsync-filter from each directory as it descends): the rule set is +loaded from a file several levels deep and must apply to that directory and +below, but not above. Also check plain --exclude / --include precedence on +files spread through the tree. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_exists, assert_not_exists, assert_same, makepath, rmtree, run_rsync, +) + +src = FROMDIR + + +def seed_ext(): + """A tree with a .log and a .txt at every level.""" + rmtree(src) + rmtree(TODIR) + cur = src + for lvl in range(4): + cur.mkdir(parents=True, exist_ok=True) + (cur / f'keep{lvl}.txt').write_text(f'txt {lvl}\n') + (cur / f'drop{lvl}.log').write_text(f'log {lvl}\n') + cur = cur / f'd{lvl + 1}' + + +# --- --exclude drops matching files at every level -------------------------- +seed_ext() +run_rsync('-a', '--exclude=*.log', f'{src}/', f'{TODIR}/') +cur = TODIR +for lvl in range(4): + assert_exists(cur / f'keep{lvl}.txt', label=f'--exclude kept txt L{lvl}') + assert_not_exists(cur / f'drop{lvl}.log', label=f'--exclude dropped log L{lvl}') + cur = cur / f'd{lvl + 1}' + +# --- --include before --exclude='*' keeps only .txt at every level ---------- +seed_ext() +run_rsync('-a', '--include=*/', '--include=*.txt', '--exclude=*', + f'{src}/', f'{TODIR}/') +cur = TODIR +for lvl in range(4): + assert_exists(cur / f'keep{lvl}.txt', label=f'--include txt L{lvl}') + assert_not_exists(cur / f'drop{lvl}.log', label=f'--include excluded log L{lvl}') + cur = cur / f'd{lvl + 1}' + +# --- -F per-directory merge file loaded from a deep directory --------------- +# .rsync-filter at d1/d2 excludes "secret*" for d1/d2 and below only. +rmtree(src) +rmtree(TODIR) +makepath(src / 'd1' / 'd2' / 'd3') +for rel in ('secret.top', 'd1/secret.mid', 'd1/d2/secret.deep', + 'd1/d2/d3/secret.deeper'): + (src / rel).write_text('x\n') +(src / 'd1' / 'd2' / '.rsync-filter').write_text('- secret*\n') + +run_rsync('-aF', f'{src}/', f'{TODIR}/') +# Above the merge file: not affected. +assert_exists(TODIR / 'secret.top', label='-F above merge dir') +assert_exists(TODIR / 'd1' / 'secret.mid', label='-F above merge dir') +# At and below the merge file: excluded. +assert_not_exists(TODIR / 'd1' / 'd2' / 'secret.deep', label='-F at merge dir') +assert_not_exists(TODIR / 'd1' / 'd2' / 'd3' / 'secret.deeper', + label='-F below merge dir') + +print("filter-depth: --exclude/--include precedence and -F per-dir merge at depth") diff --git a/testsuite/size-filter_test.py b/testsuite/size-filter_test.py new file mode 100644 index 000000000..ce5286893 --- /dev/null +++ b/testsuite/size-filter_test.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Coverage of --max-size / --min-size at depth. + +A small and a large file at every level; --max-size must transfer only the +small ones and --min-size only the large ones, the selection holding all the +way down the tree. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_exists, assert_not_exists, make_data_file, rmtree, run_rsync, +) + +src = FROMDIR +SMALL = 500 +LARGE = 5000 + + +def seed(): + rmtree(src) + rmtree(TODIR) + cur = src + for lvl in range(4): + cur.mkdir(parents=True, exist_ok=True) + make_data_file(cur / f'small{lvl}', SMALL) + make_data_file(cur / f'large{lvl}', LARGE) + cur = cur / f'd{lvl + 1}' + + +# --- --max-size keeps only the small files at every level ------------------- +seed() +run_rsync('-a', '--max-size=1000', f'{src}/', f'{TODIR}/') +cur = TODIR +for lvl in range(4): + assert_exists(cur / f'small{lvl}', label=f'--max-size kept small L{lvl}') + assert_not_exists(cur / f'large{lvl}', label=f'--max-size dropped large L{lvl}') + cur = cur / f'd{lvl + 1}' + +# --- --min-size keeps only the large files at every level ------------------- +seed() +run_rsync('-a', '--min-size=1000', f'{src}/', f'{TODIR}/') +cur = TODIR +for lvl in range(4): + assert_exists(cur / f'large{lvl}', label=f'--min-size kept large L{lvl}') + assert_not_exists(cur / f'small{lvl}', label=f'--min-size dropped small L{lvl}') + cur = cur / f'd{lvl + 1}' + +print("size-filter: --max-size / --min-size select correctly at depth") From 9bff44ebde2110aa2fd85c8d0bf89e8fd151e028 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 07:59:23 +1000 Subject: [PATCH 06/15] testsuite: daemon parameter coverage (loopback) Drive a loopback daemon (secure stdio-pipe transport by default, also green under --use-tcp) via the new write_daemon_conf helper and assert the behaviour of the security-relevant rsyncd.conf parameters, transferring >=3-deep trees: daemon-access path / read only / write only / list, incl. a deep sub-path pull and that a list=no module is hidden yet usable by name. daemon-filter daemon exclude hides matching files everywhere; incoming / outgoing chmod rewrite modes of every transferred file. daemon-auth auth users + secrets file accept the right password, reject a wrong one and an unauthenticated request; strict modes rejects a world-readable secrets file. daemon-exec pre-/post-xfer exec run with RSYNC_MODULE_NAME / RSYNC_EXIT_STATUS; a failing pre-xfer exec aborts the transfer (marker files polled for, since post-xfer exec runs after the client disconnects under TCP). daemon-munge munge symlinks stores incoming links with the /rsyncd-munged/ prefix and strips it on the way out. daemon-refuse refuse options: a named option, a wildcard, and the '* !a !v' allow-list idiom. Green on master under pipe and --use-tcp transports and under --protocol=29. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/daemon-access_test.py | 82 +++++++++++++++++++++++++++++ testsuite/daemon-auth_test.py | 92 +++++++++++++++++++++++++++++++++ testsuite/daemon-exec_test.py | 84 ++++++++++++++++++++++++++++++ testsuite/daemon-filter_test.py | 76 +++++++++++++++++++++++++++ testsuite/daemon-munge_test.py | 53 +++++++++++++++++++ testsuite/daemon-refuse_test.py | 74 ++++++++++++++++++++++++++ 6 files changed, 461 insertions(+) create mode 100644 testsuite/daemon-access_test.py create mode 100644 testsuite/daemon-auth_test.py create mode 100644 testsuite/daemon-exec_test.py create mode 100644 testsuite/daemon-filter_test.py create mode 100644 testsuite/daemon-munge_test.py create mode 100644 testsuite/daemon-refuse_test.py diff --git a/testsuite/daemon-access_test.py b/testsuite/daemon-access_test.py new file mode 100644 index 000000000..fe07a4936 --- /dev/null +++ b/testsuite/daemon-access_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Daemon coverage: module path, read only, write only, list. + +Drives a loopback daemon (secure stdio-pipe transport by default) and checks +the access-control parameters, transferring a >=3-deep tree through each module +and pulling a deep sub-path to exercise in-module path resolution. +""" + +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + assert_same, make_tree, makepath, rmtree, rsync_argv, run_rsync, + start_test_daemon, test_fail, verify_dirs, walk_files, write_daemon_conf, +) + +DAEMON_PORT = 12886 + +src = FROMDIR +rwdir = SCRATCHDIR / 'rwdest' +wodir = SCRATCHDIR / 'wodest' +pulld = SCRATCHDIR / 'pulled' +for d in (rwdir, wodir, pulld, TODIR): + rmtree(d) +makepath(rwdir, wodir) +rmtree(src) +make_tree(src, depth=3) +rels = [p.relative_to(src) for p in walk_files(src)] + +conf = write_daemon_conf([ + ('ro', {'path': src, 'read only': 'yes', 'comment': 'r/o'}), + ('rw', {'path': rwdir, 'read only': 'no', 'comment': 'r/w'}), + ('wo', {'path': wodir, 'read only': 'no', 'write only': 'yes'}), + ('hidden', {'path': src, 'list': 'no'}), +]) +url = start_test_daemon(conf, DAEMON_PORT) + + +def fails(args, what): + proc = subprocess.run(rsync_argv(*args), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) + if proc.returncode == 0: + test_fail(f"{what} unexpectedly succeeded") + return proc.stderr + + +# --- read only module: pull works (deep), push refused ---------------------- +run_rsync('-a', f'{url}ro/', f'{pulld}/', check=False) # codes 0/23 ok +for rel in rels: + assert_same(pulld / rel, src / rel, label=f'pull ro {rel}') +# deep sub-path pull +rmtree(pulld) +makepath(pulld) +run_rsync('-a', f'{url}ro/d1/d2/', f'{pulld}/', check=False) +assert_same(pulld / 'f2', src / 'd1' / 'd2' / 'f2', label='deep sub-path pull') +fails(['-a', f'{src}/', f'{url}ro/'], "push to a read-only module") + +# --- read/write module: push works at depth --------------------------------- +run_rsync('-a', f'{src}/', f'{url}rw/', check=False) +verify_dirs(src, rwdir, label="push to rw module") + +# --- write only module: push works, pull refused ---------------------------- +run_rsync('-a', f'{src}/', f'{url}wo/', check=False) +verify_dirs(src, wodir, label="push to wo module") +fails(['-a', f'{url}wo/', f'{pulld}/'], "pull from a write-only module") + +# --- list: hidden module absent from the listing, ro/rw/wo present ---------- +proc = subprocess.run(rsync_argv(url), capture_output=True, text=True) +listing = proc.stdout +for m in ('ro', 'rw', 'wo'): + if m not in listing: + test_fail(f"module {m} missing from the daemon listing:\n{listing}") +if 'hidden' in listing: + test_fail(f"list=no module leaked into the listing:\n{listing}") +# ...but the hidden module is still usable by name. +rmtree(pulld) +makepath(pulld) +run_rsync('-a', f'{url}hidden/f0', f'{pulld}/', check=False) +assert_same(pulld / 'f0', src / 'f0', label='hidden module usable by name') + +print("daemon-access: read only / write only / list / deep paths verified") diff --git a/testsuite/daemon-auth_test.py b/testsuite/daemon-auth_test.py new file mode 100644 index 000000000..5075f1cbf --- /dev/null +++ b/testsuite/daemon-auth_test.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Daemon coverage: auth users, secrets file, strict modes. + +A module with auth users + a secrets file must accept the right password, +reject a wrong one, and (with the default strict modes) refuse a +world-readable secrets file. Authentication happens in the daemon protocol, so +it works over the default secure stdio-pipe transport. +""" + +import os +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + make_tree, makepath, rmtree, rsync_argv, start_test_daemon, test_fail, + verify_dirs, write_daemon_conf, +) + +DAEMON_PORT = 12888 + +# When a daemon module needs auth and no password is available, rsync falls back +# to an interactive getpass() prompt that reads /dev/tty directly -- which the +# test harness cannot redirect, so it would hang `make coverage` (or any run +# with a controlling terminal). Give every client a fallback password via the +# environment so it never prompts: the --password-file cases below override it, +# and the invalid-credentials case uses it and is correctly rejected. +os.environ['RSYNC_PASSWORD'] = 'env-fallback-wrong' + +src = FROMDIR +rmtree(src) +make_tree(src, depth=3) + +authdir = SCRATCHDIR / 'authdest' +secrets = SCRATCHDIR / 'rsyncd.secrets' +secrets.write_text('tuser:secretpass\n') +secrets.chmod(0o600) + +conf = write_daemon_conf([ + ('auth', {'path': authdir, 'read only': 'no', + 'auth users': 'tuser', 'secrets file': secrets}), +]) +url = start_test_daemon(conf, DAEMON_PORT) +userurl = url.replace('rsync://', 'rsync://tuser@', 1) + + +def pwfile(name, text): + p = SCRATCHDIR / name + p.write_text(text) + p.chmod(0o600) + return p + + +def push(pw, **kw): + rmtree(authdir) + makepath(authdir) + return subprocess.run( + rsync_argv('-a', f'--password-file={pw}', f'{src}/', f'{userurl}auth/'), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, **kw) + + +# --- correct password succeeds ---------------------------------------------- +ok = pwfile('pw.ok', 'secretpass\n') +proc = push(ok) +if proc.returncode not in (0, 23): + test_fail(f"auth with the correct password failed: {proc.stderr}") +verify_dirs(src, authdir, label="auth success") + +# --- wrong password is rejected --------------------------------------------- +bad = pwfile('pw.bad', 'wrongpass\n') +proc = push(bad) +if proc.returncode == 0: + test_fail("auth with the wrong password unexpectedly succeeded") + +# --- a request with invalid credentials is rejected ------------------------ +# Local user (not an auth user) with the wrong env-supplied password; rejected +# without ever prompting on the tty. +proc = subprocess.run( + rsync_argv('-a', f'{src}/', f'{url}auth/'), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, + stdin=subprocess.DEVNULL) +if proc.returncode == 0: + test_fail("a request with invalid credentials succeeded against an " + "auth-users module") + +# --- strict modes rejects a world-readable secrets file --------------------- +secrets.chmod(0o644) +proc = push(ok) +if proc.returncode == 0: + test_fail("strict modes did not reject a world-readable secrets file") +secrets.chmod(0o600) + +print("daemon-auth: auth users / secrets file / strict modes verified") diff --git a/testsuite/daemon-exec_test.py b/testsuite/daemon-exec_test.py new file mode 100644 index 000000000..6ac80a3e4 --- /dev/null +++ b/testsuite/daemon-exec_test.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Daemon coverage: pre-xfer exec and post-xfer exec. + +A module's pre-xfer/post-xfer exec hooks must run with the documented +environment (RSYNC_MODULE_NAME, RSYNC_EXIT_STATUS, ...), and a non-zero +pre-xfer exec must abort the transfer. +""" + +import subprocess +import time + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + make_tree, makepath, rmtree, rsync_argv, start_test_daemon, test_fail, + write_daemon_conf, +) + + +def wait_for(path, want, secs=5): + """Poll for a marker file to contain `want`. post-xfer exec runs on the + daemon side after the client disconnects (a real race under --use-tcp), + so we must wait for it rather than check immediately.""" + deadline = time.monotonic() + secs + while time.monotonic() < deadline: + if path.is_file() and path.read_text().strip() == want: + return True + time.sleep(0.05) + return False + +DAEMON_PORT = 12889 + +src = FROMDIR +rmtree(src) +make_tree(src, depth=3) + +markers = SCRATCHDIR / 'markers' +rmtree(markers) +makepath(markers) +hookdir = SCRATCHDIR / 'hookdest' +faildir = SCRATCHDIR / 'faildest' +makepath(hookdir, faildir) + + +def script(name, body): + p = SCRATCHDIR / name + p.write_text('#!/bin/sh\n' + body) + p.chmod(0o755) + return p + + +pre = script('pre.sh', f'echo "$RSYNC_MODULE_NAME" > {markers}/pre.out\nexit 0\n') +post = script('post.sh', f'echo "$RSYNC_EXIT_STATUS" > {markers}/post.out\n' + 'exit 0\n') +prefail = script('prefail.sh', 'exit 1\n') + +conf = write_daemon_conf([ + ('hook', {'path': hookdir, 'read only': 'no', + 'pre-xfer exec': pre, 'post-xfer exec': post}), + ('failhook', {'path': faildir, 'read only': 'no', + 'pre-xfer exec': prefail}), +]) +url = start_test_daemon(conf, DAEMON_PORT) + +# --- pre/post hooks run with the documented environment --------------------- +proc = subprocess.run(rsync_argv('-a', f'{src}/', f'{url}hook/'), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) +if proc.returncode not in (0, 23): + test_fail(f"transfer through exec-hook module failed: {proc.stderr}") +if not wait_for(markers / 'pre.out', 'hook'): + test_fail("pre-xfer exec did not run with RSYNC_MODULE_NAME=hook") +if not wait_for(markers / 'post.out', '0'): + test_fail("post-xfer exec did not run with RSYNC_EXIT_STATUS=0") + +# --- a failing pre-xfer exec aborts the transfer ---------------------------- +proc = subprocess.run(rsync_argv('-a', f'{src}/', f'{url}failhook/'), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) +if proc.returncode == 0: + test_fail("a failing pre-xfer exec did not abort the transfer") +if list(faildir.iterdir()): + test_fail("transfer wrote files despite a failing pre-xfer exec") + +print("daemon-exec: pre-xfer/post-xfer exec env + abort-on-failure verified") diff --git a/testsuite/daemon-filter_test.py b/testsuite/daemon-filter_test.py new file mode 100644 index 000000000..cac387e8b --- /dev/null +++ b/testsuite/daemon-filter_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Daemon coverage: exclude, incoming chmod, outgoing chmod (at depth). + +A daemon-side exclude must hide matching files everywhere in the module tree; +incoming/outgoing chmod must rewrite the permissions of every transferred file, +including ones several levels deep. +""" + +import os +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + assert_mode, assert_not_exists, assert_same, make_tree, makepath, rmtree, + rsync_argv, start_test_daemon, test_fail, walk_files, write_daemon_conf, +) + +DAEMON_PORT = 12887 + +src = FROMDIR +rmtree(src) +make_tree(src, depth=3) +(src / 'a.secret').write_text('s\n') +(src / 'd1' / 'd2' / 'b.secret').write_text('s\n') +rels = [p.relative_to(src) for p in walk_files(src)] + +incdir = SCRATCHDIR / 'incdest' +for d in (incdir,): + rmtree(d) +makepath(incdir) + +conf = write_daemon_conf([ + ('filt', {'path': src, 'read only': 'yes', 'exclude': '*.secret'}), + ('inc', {'path': incdir, 'read only': 'no', 'incoming chmod': 'F600'}), + ('out', {'path': src, 'read only': 'yes', 'outgoing chmod': 'Fg-r,Fo-r'}), +]) +url = start_test_daemon(conf, DAEMON_PORT) + + +def pull(mod, dest): + rmtree(dest) + makepath(dest) + subprocess.run(rsync_argv('-a', f'{url}{mod}/', f'{dest}/'), + stdout=subprocess.DEVNULL) + + +# --- daemon exclude hides *.secret everywhere in the module ----------------- +fp = SCRATCHDIR / 'filtpull' +pull('filt', fp) +assert_not_exists(fp / 'a.secret', label='daemon exclude top') +assert_not_exists(fp / 'd1' / 'd2' / 'b.secret', label='daemon exclude deep') +assert_same(fp / 'd1' / 'd2' / 'f2', src / 'd1' / 'd2' / 'f2', + label='daemon exclude kept others') + +# --- incoming chmod rewrites pushed file modes at depth --------------------- +subprocess.run(rsync_argv('-a', f'{src}/', f'{url}inc/'), + stdout=subprocess.DEVNULL) +checked = 0 +for rel in rels: + p = incdir / rel + if p.is_file(): + assert_mode(p, 0o600, label=f'incoming chmod {rel}') + checked += 1 +if checked == 0: + test_fail("incoming chmod test transferred no files") + +# --- outgoing chmod rewrites pulled file modes at depth --------------------- +op = SCRATCHDIR / 'outpull' +pull('out', op) +for rel in rels: + p = op / rel + if p.is_file() and (os.stat(p).st_mode & 0o044): + test_fail(f"outgoing chmod did not clear group/other read on {rel}: " + f"{oct(os.stat(p).st_mode & 0o777)}") + +print("daemon-filter: exclude / incoming chmod / outgoing chmod verified at depth") diff --git a/testsuite/daemon-munge_test.py b/testsuite/daemon-munge_test.py new file mode 100644 index 000000000..05fdb4695 --- /dev/null +++ b/testsuite/daemon-munge_test.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Daemon coverage: munge symlinks. + +A module with "munge symlinks = yes" stores incoming symlinks with a +/rsyncd-munged/ prefix (so they can't be used to escape the module) and strips +that prefix from outgoing symlinks. Verify both directions on a symlink several +levels deep. +""" + +import os +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + assert_is_symlink, make_tree, makepath, rmtree, rsync_argv, + start_test_daemon, test_fail, write_daemon_conf, +) + +DAEMON_PORT = 12890 + +src = FROMDIR +deep = os.path.join('d1', 'd2') +rmtree(src) +make_tree(src, depth=3) +os.symlink('f3', src / deep / 'sl') # deep symlink -> f3 + +mungedest = SCRATCHDIR / 'mungedest' +pulled = SCRATCHDIR / 'mungepull' +for d in (mungedest, pulled): + rmtree(d) +makepath(mungedest, pulled) + +conf = write_daemon_conf([ + ('munge', {'path': mungedest, 'read only': 'no', 'munge symlinks': 'yes'}), +]) +url = start_test_daemon(conf, DAEMON_PORT) + +# --- push: the stored symlink is munged with the /rsyncd-munged/ prefix ------ +subprocess.run(rsync_argv('-al', f'{src}/', f'{url}munge/'), + stdout=subprocess.DEVNULL) +stored = mungedest / deep / 'sl' +assert_is_symlink(stored, label='munge stored symlink') +target = os.readlink(stored) +if target != '/rsyncd-munged/f3': + test_fail(f"munge symlinks stored {target!r}, expected '/rsyncd-munged/f3'") + +# --- pull: the prefix is stripped back off on the way out ------------------- +subprocess.run(rsync_argv('-al', f'{url}munge/', f'{pulled}/'), + stdout=subprocess.DEVNULL) +out = pulled / deep / 'sl' +assert_is_symlink(out, target='f3', label='munge stripped on pull') + +print("daemon-munge: munge symlinks adds/strips /rsyncd-munged/ at depth") diff --git a/testsuite/daemon-refuse_test.py b/testsuite/daemon-refuse_test.py new file mode 100644 index 000000000..770b36ba9 --- /dev/null +++ b/testsuite/daemon-refuse_test.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Daemon coverage: refuse options (a named option, a wildcard, and the +allow-list negation form). + +daemon-refuse-compress_test.py covers the basic case; this widens it to a +different named option, a wildcard pattern, and the "* !a !v" allow-list idiom +documented in rsyncd.conf.5. +""" + +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + make_tree, makepath, rmtree, rsync_argv, run_rsync, start_test_daemon, + test_fail, verify_dirs, write_daemon_conf, +) + +DAEMON_PORT = 12891 + +src = FROMDIR +rmtree(src) +make_tree(src, depth=3) + +deldir = SCRATCHDIR / 'deldest' +makepath(deldir) + +conf = write_daemon_conf([ + ('refuse-delete', {'path': deldir, 'read only': 'no', + 'refuse options': 'delete'}), + ('refuse-wild', {'path': src, 'read only': 'yes', + 'refuse options': 'checksum*'}), + ('only-av', {'path': src, 'read only': 'yes', + 'refuse options': '* !a !v'}), +]) +url = start_test_daemon(conf, DAEMON_PORT) + + +def refused(args, what): + proc = subprocess.run(rsync_argv(*args), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) + if proc.returncode == 0: + test_fail(f"{what} was not refused") + return proc.stderr + + +def allowed(args, what): + proc = subprocess.run(rsync_argv(*args), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) + if proc.returncode not in (0, 23): + test_fail(f"{what} was unexpectedly refused: {proc.stderr}") + + +# --- a named refused option (delete) ---------------------------------------- +refused(['-a', '--delete', f'{src}/', f'{url}refuse-delete/'], + "--delete on a refuse=delete module") +allowed(['-a', f'{src}/', f'{url}refuse-delete/'], + "plain push to a refuse=delete module") + +# --- a wildcard refused option (checksum*) ---------------------------------- +dest = SCRATCHDIR / 'wilddest' +makepath(dest) +refused(['-a', '--checksum', f'{url}refuse-wild/', f'{dest}/'], + "--checksum on a refuse=checksum* module") + +# --- the "* !a !v" allow-list: -av allowed, -z refused ---------------------- +rmtree(dest) +makepath(dest) +allowed(['-av', f'{url}only-av/', f'{dest}/'], "-av on an allow-list module") +refused(['-avz', f'{url}only-av/', f'{dest}/'], + "-z on an allow-list module") + +print("daemon-refuse: named / wildcard / allow-list refuse options verified") From 29d8f2b7bdc1ac027887b8bac825289a6b45326e Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 08:01:52 +1000 Subject: [PATCH 07/15] testsuite: output, comparison and algorithm-selection option coverage Breadth pass for options not yet exercised: output-options output shape of --version/--help/-i/-n/--stats/ --out-format/--list-only/-q/--progress/-h/-8 (these control output, not path handling, so they're checked for shape). compare -c and -I catch a stealth change (same size+mtime, new content) deep in the tree; --size-only skips a same-size change; --modify-window absorbs a 1s mtime difference. compress-options --compress-choice for every advertised compressor, --compress-level, --skip-compress, --checksum-choice for every advertised checksum, and --checksum-seed -- each a clean byte-identical transfer at depth. Green on master and under --protocol=29/30. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/compare_test.py | 90 ++++++++++++++++++++++++++++++ testsuite/compress-options_test.py | 65 +++++++++++++++++++++ testsuite/output-options_test.py | 90 ++++++++++++++++++++++++++++++ 3 files changed, 245 insertions(+) create mode 100644 testsuite/compare_test.py create mode 100644 testsuite/compress-options_test.py create mode 100644 testsuite/output-options_test.py diff --git a/testsuite/compare_test.py b/testsuite/compare_test.py new file mode 100644 index 000000000..8279c9ff4 --- /dev/null +++ b/testsuite/compare_test.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Coverage of the comparison/skip options at depth: -c, -I, --size-only, +--modify-window. + +These decide WHETHER a file is transferred. Each is checked on a file several +levels deep using a "stealth change" (same size, same mtime, different content) +that the default quick check deliberately skips. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, make_tree, rmtree, run_rsync, test_fail, +) + +src = FROMDIR +deep = os.path.join('d1', 'd2', 'd3', 'f3') + + +def seed(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3, data=True, data_size=4096) + run_rsync('-a', f'{src}/', f'{TODIR}/') # dest == src + + +def stealth_change(): + """Change the deep source file's content but restore the destination's + size+mtime, so the quick check sees them as equal.""" + st = os.stat(TODIR / deep) + data = bytearray((src / deep).read_bytes()) + data[0] ^= 0xFF # same length, new content + (src / deep).write_bytes(bytes(data)) + os.utime(src / deep, (st.st_atime, st.st_mtime)) + + +# --- the default quick check skips a stealth change; -c and -I catch it ------ +seed() +stealth_change() +run_rsync('-a', f'{src}/', f'{TODIR}/') +if (TODIR / deep).read_bytes() == (src / deep).read_bytes(): + test_fail("default quick check unexpectedly transferred a same-size, " + "same-mtime change (test setup is wrong)") + +run_rsync('-a', '-c', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='-c caught stealth change') + +seed() +stealth_change() +run_rsync('-a', '-I', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='-I caught stealth change') + +# --- --size-only skips a same-size change even when the mtime differs -------- +def samesize_newmtime(): + data = bytearray((src / deep).read_bytes()) + data[0] ^= 0xFF # same size, new content + (src / deep).write_bytes(bytes(data)) + st = os.stat(src / deep) + os.utime(src / deep, (st.st_atime, st.st_mtime + 100)) # mtime differs + + +seed() +samesize_newmtime() +run_rsync('-a', '--size-only', f'{src}/', f'{TODIR}/') +if (TODIR / deep).read_bytes() == (src / deep).read_bytes(): + test_fail("--size-only transferred a same-size file (should have skipped)") + +# Contrast on a fresh tree: the default DOES transfer it (mtime differs). +# (Re-seed because --size-only above updated the dest mtime to match.) +seed() +samesize_newmtime() +run_rsync('-a', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='default caught mtime change') + +# --- --modify-window absorbs a small mtime difference ----------------------- +# Both runs are dry-run (-ain): a real run would update the dest mtime to match +# the source, leaving the --modify-window run nothing to absorb (vacuous). +seed() +st = os.stat(TODIR / deep) +os.utime(src / deep, (st.st_atime, st.st_mtime + 1)) # 1s newer, same content +p = run_rsync('-ain', f'{src}/', f'{TODIR}/', capture_output=True) +if 'f3' not in p.stdout: + test_fail("a 1s mtime change was not itemized without --modify-window") +p = run_rsync('-ain', '--modify-window=2', f'{src}/', f'{TODIR}/', + capture_output=True) +if 'f3' in p.stdout: + test_fail("--modify-window=2 did not absorb a 1s mtime difference") + +print("compare: -c / -I / --size-only / --modify-window verified at depth") diff --git a/testsuite/compress-options_test.py b/testsuite/compress-options_test.py new file mode 100644 index 000000000..74e087712 --- /dev/null +++ b/testsuite/compress-options_test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Breadth coverage of the algorithm-selection options at depth: +--compress-choice / --compress-level / --skip-compress and +--checksum-choice / --checksum-seed. + +Compression and checksum choice don't change the result, so each available +algorithm is exercised for a clean, byte-identical transfer of a >=3-deep tree +(proving the option parses, negotiates and doesn't corrupt data). +""" + +import json + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, make_tree, rmtree, run_rsync, walk_files, +) + +src = FROMDIR +vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout) +compressors = [a for a in vv.get('compress_list', []) if a != 'none'] +checksums = [a for a in vv.get('checksum_list', []) if a != 'none'] + + +def fresh(): + rmtree(src) + rmtree(TODIR) + make_tree(src, depth=3, data=True, data_size=4096) + return [p.relative_to(src) for p in walk_files(src)] + + +def verify(rels, label): + for rel in rels: + assert_same(TODIR / rel, src / rel, label=f'{label} {rel}') + + +# --- --compress-choice for every advertised compressor ---------------------- +for algo in compressors: + rels = fresh() + run_rsync('-az', f'--compress-choice={algo}', f'{src}/', f'{TODIR}/') + verify(rels, f'--compress-choice={algo}') + +# --- --compress-level ------------------------------------------------------- +rels = fresh() +run_rsync('-az', '--compress-level=9', f'{src}/', f'{TODIR}/') +verify(rels, '--compress-level=9') + +# --- --skip-compress (the file must still arrive intact) -------------------- +rels = fresh() +(src / 'd1' / 'd2' / 'x.gz').write_bytes(b'\x1f\x8b' + b'pseudo gzip body ' * 64) +run_rsync('-az', '--skip-compress=gz', f'{src}/', f'{TODIR}/') +assert_same(TODIR / 'd1' / 'd2' / 'x.gz', src / 'd1' / 'd2' / 'x.gz', + label='--skip-compress gz') + +# --- --checksum-choice for every advertised checksum ------------------------ +for algo in checksums: + rels = fresh() + run_rsync('-a', '-c', f'--checksum-choice={algo}', f'{src}/', f'{TODIR}/') + verify(rels, f'--checksum-choice={algo}') + +# --- --checksum-seed -------------------------------------------------------- +rels = fresh() +run_rsync('-a', '-c', '--checksum-seed=12345', f'{src}/', f'{TODIR}/') +verify(rels, '--checksum-seed') + +print("compress-options: compress/checksum algorithm selection verified at depth") diff --git a/testsuite/output-options_test.py b/testsuite/output-options_test.py new file mode 100644 index 000000000..3a836e1cd --- /dev/null +++ b/testsuite/output-options_test.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Breadth coverage of the output / reporting options. + +These options control rsync's OUTPUT, not its path handling, so they are +checked for the documented output shape rather than at depth: + --version, --help, --itemize-changes (-i), --dry-run (-n), --stats, + --out-format, --list-only, --quiet (-q), --progress, -h, -8. +""" + +import subprocess + +from rsyncfns import ( + FROMDIR, TODIR, + assert_not_exists, make_tree, rmtree, rsync_argv, test_fail, +) + +src = FROMDIR + + +def out(*args): + return subprocess.run(rsync_argv(*args), capture_output=True, text=True) + + +# --- --version / --help ----------------------------------------------------- +p = out('--version') +if p.returncode != 0 or 'protocol version' not in p.stdout: + test_fail(f"--version output unexpected:\n{p.stdout}") +p = out('--help') +help_txt = p.stdout + p.stderr +if 'rsync' not in help_txt or 'Usage' not in help_txt: + test_fail("--help did not print usage") + +rmtree(src) +rmtree(TODIR) +make_tree(src, depth=2) + +# --- --itemize-changes: a new file shows the create itemization ------------- +p = out('-ai', f'{src}/', f'{TODIR}/') +if '>f+++++++++' not in p.stdout: + test_fail(f"--itemize-changes missing create line:\n{p.stdout}") + +# --- --dry-run lists but does not create ------------------------------------ +rmtree(TODIR) +p = out('-ain', f'{src}/', f'{TODIR}/') +if '>f+++++++++' not in p.stdout: + test_fail("--dry-run itemize output missing") +assert_not_exists(TODIR / 'f0', label='--dry-run created a file') + +# --- --stats prints the summary block --------------------------------------- +rmtree(TODIR) +p = out('-a', '--stats', f'{src}/', f'{TODIR}/') +if 'Number of files:' not in p.stdout or 'Total file size:' not in p.stdout: + test_fail(f"--stats output missing expected lines:\n{p.stdout}") + +# --- --out-format=%n emits bare filenames ----------------------------------- +rmtree(TODIR) +p = out('-a', '--out-format=%n', f'{src}/', f'{TODIR}/') +if 'f0' not in p.stdout: + test_fail(f"--out-format=%n did not emit filenames:\n{p.stdout}") + +# --- --list-only lists the source without copying --------------------------- +# Pass a destination too: without --list-only this transfer would populate +# TODIR, so the assert_not_exists below actually proves the "without copying" +# property rather than being vacuously true for a destination-less command. +rmtree(TODIR) +p = out('--list-only', '-r', f'{src}/', f'{TODIR}/') +if 'f0' not in p.stdout: + test_fail(f"--list-only did not list files:\n{p.stdout}") +assert_not_exists(TODIR / 'f0', label='--list-only copied a file') + +# --- --quiet suppresses normal stdout --------------------------------------- +rmtree(TODIR) +p = out('-a', '-q', f'{src}/', f'{TODIR}/') +if p.stdout.strip() != '': + test_fail(f"--quiet produced stdout: {p.stdout!r}") + +# --- --progress shows a percentage ------------------------------------------ +rmtree(TODIR) +p = out('-a', '--progress', f'{src}/', f'{TODIR}/') +if '100%' not in p.stdout: + test_fail(f"--progress did not show a percentage:\n{p.stdout}") + +# --- -h / -8 do not break a transfer ---------------------------------------- +rmtree(TODIR) +p = out('-a', '-h', '-8', '--stats', f'{src}/', f'{TODIR}/') +if p.returncode != 0: + test_fail(f"-h/-8 broke the transfer:\n{p.stderr}") + +print("output-options: version/help/-i/-n/--stats/--out-format/--list-only/" + "-q/--progress/-h/-8 verified") From 9f8348e67eba8e632502f85c688271bee20f2c2a Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 08:12:39 +1000 Subject: [PATCH 08/15] testsuite: probe RESOLVE_BENEATH support functionally for the #715 test Add resolve_beneath_supported() to rsyncfns: it functionally probes whether the rsync binary can follow an in-tree directory symlink under its secure resolver (an initial transfer plus a delta update through a dir-symlink, the operation issue #715 is about). This tracks the actual binary instead of a platform name. Use it in symlink-dirlink-basis_test.py in place of the SunOS/OpenBSD/NetBSD/ Cygwin name check: it skips on those platforms too, and additionally on Linux < 5.6, a seccomp-blocked openat2, and the new --disable-openat2 build, where the portable O_NOFOLLOW fallback rejects the in-tree symlink. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/rsyncfns.py | 42 +++++++++++++++++++++++++ testsuite/symlink-dirlink-basis_test.py | 17 +++++++--- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/testsuite/rsyncfns.py b/testsuite/rsyncfns.py index b9bcbf17b..3a3b37b19 100644 --- a/testsuite/rsyncfns.py +++ b/testsuite/rsyncfns.py @@ -1126,6 +1126,48 @@ def assert_not_exists(path, label: str = '') -> 'None': test_fail(f"{_tag(label)}{path} exists but should not") +_rb_cache = None + + +def resolve_beneath_supported() -> bool: + """True if this rsync can FOLLOW an in-tree directory symlink under its + secure resolver -- i.e. update a file through a dir-symlink on the receiver + (--keep-dirlinks; issue #715). + + False wherever the portable per-component O_NOFOLLOW fallback is the active + resolver: a platform with no kernel "beneath" primitive, Linux < 5.6, a + seccomp-blocked openat2, or a --disable-openat2 build. There the delta + update through the symlinked directory fails verification. Probed + functionally (an initial transfer plus a delta update through a dir-symlink) + so it tracks the actual binary rather than a platform name, and cached.""" + global _rb_cache + if _rb_cache is not None: + return _rb_cache + probe = SCRATCHDIR / '.rb_probe' + rmtree(probe) + (probe / 'home' / 'real').mkdir(parents=True) + os.symlink('real', probe / 'home' / 'link') + (probe / 'src' / 'link').mkdir(parents=True) + f = probe / 'src' / 'link' / 'f' + make_data_file(f, 40000) + + def push(): + subprocess.run( + rsync_argv('-KRl', '--no-whole-file', 'link/f', + f"{probe / 'home'}/"), + cwd=str(probe / 'src'), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + push() + with open(f, 'ab') as fh: # size change -> forces a delta update + fh.write(b'appended tail for delta\n') + push() + dst = probe / 'home' / 'real' / 'f' + _rb_cache = dst.is_file() and filecmp.cmp(str(f), str(dst), shallow=False) + rmtree(probe) + return _rb_cache + + def write_daemon_conf(modules, globals=None, *, name: str = 'test-rsyncd.conf') -> 'Path': """Write a custom rsyncd.conf for daemon-parameter tests. diff --git a/testsuite/symlink-dirlink-basis_test.py b/testsuite/symlink-dirlink-basis_test.py index b952b4de2..fd880144a 100644 --- a/testsuite/symlink-dirlink-basis_test.py +++ b/testsuite/symlink-dirlink-basis_test.py @@ -14,20 +14,27 @@ import filecmp import os -import platform import subprocess import time from rsyncfns import ( RSYNC, SCRATCHDIR, SRCDIR, TMPDIR, - make_data_file, rsync_argv, test_fail, test_skipped, + make_data_file, resolve_beneath_supported, rsync_argv, test_fail, + test_skipped, ) -if platform.system() in ('SunOS', 'OpenBSD', 'NetBSD') or platform.system().startswith('CYGWIN'): +# Following an in-tree directory symlink under the secure resolver needs a +# kernel "beneath" primitive (RESOLVE_BENEATH/O_RESOLVE_BENEATH). Probe the +# actual binary rather than guessing from the platform name: this also skips +# correctly on Linux < 5.6, a seccomp-blocked openat2, and a --disable-openat2 +# build, where the portable O_NOFOLLOW fallback rejects the in-tree symlink +# (issue #715). +if not resolve_beneath_supported(): test_skipped( - f"secure_relative_open lacks RESOLVE_BENEATH equivalent on " - f"{platform.system()}; issue #715 still affects this platform" + "this rsync can't follow an in-tree directory symlink under its " + "secure resolver (no RESOLVE_BENEATH equivalent / --disable-openat2); " + "issue #715 still affects this configuration" ) os.environ['RSYNC_RSH'] = str(SRCDIR / 'support' / 'lsh.sh') From e561174587decf53598ae58e85e7738450c6a7d0 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 08:12:39 +1000 Subject: [PATCH 09/15] build: add gcov coverage and --disable-openat2 knobs for the test suite Two test-coverage build knobs (both behaviour-neutral by default): --enable-coverage appends '--coverage -fprofile-update=atomic -O0' and adds a 'make coverage' target (whole suite, run serially, then gcovr HTML with branch + decision coverage). rsync forks and its children exit without running the gcov atexit flush -- the generator via its SIGUSR1 handler (_exit_cleanup) and the receiver via the SIGUSR2 handler -- so under GCOV_COVERAGE we call __gcov_dump() at both, or receiver.c/generator.c record no coverage at all. --disable-openat2 gates the Linux openat2(RESOLVE_BENEATH) sites in syscall.c on HAVE_OPENAT2 (defined by default), so disabling it forces the portable per-component O_NOFOLLOW resolver to run as the primary on ordinary Linux -- exercising and coverage-counting that fallback tier without a pre-5.6 kernel. NOTE: coordinate with the parallel syscall.c path-resolution restructure. Co-Authored-By: Claude Opus 4.7 (1M context) --- Makefile.in | 34 ++++++++++++++++++++++++++++++++++ cleanup.c | 10 +++++++++- configure.ac | 26 ++++++++++++++++++++++++++ main.c | 5 +++++ syscall.c | 10 ++++++---- 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/Makefile.in b/Makefile.in index af9fbfb28..8f3b04c9d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -280,6 +280,8 @@ clean: cleantests rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) @MAKE_RRSYNC@ \ git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 @MAKE_RRSYNC_1@ \ *.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp + rm -f *.gcno *.gcda lib/*.gcno lib/*.gcda zlib/*.gcno zlib/*.gcda popt/*.gcno popt/*.gcda + rm -rf coverage .PHONY: cleantests cleantests: @@ -324,6 +326,15 @@ test: check # `make check CHECK_J=1` (serial) or any other value. CHECK_J = 8 +# Parallelism for `make coverage`. Defaults to the same as CHECK_J: the +# coverage build sets -fprofile-update=atomic (atomic in-memory counters) and +# gcc's libgcov serializes the per-source .gcda read-modify-write merge with a +# file lock, so concurrent rsync processes (incl. the forked sender/generator/ +# receiver) accumulate exactly -- verified by a count-linearity check (a hot +# line accumulates identically at -j1 and -P16). Override with +# `make coverage COVERAGE_J=1` if your libgcov does not lock .gcda merges. +COVERAGE_J = $(CHECK_J) + .PHONY: check check: all $(CHECK_PROGS) $(CHECK_SYMLINKS) $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(CHECK_J) @@ -336,6 +347,29 @@ check29: all $(CHECK_PROGS) $(CHECK_SYMLINKS) check30: all $(CHECK_PROGS) $(CHECK_SYMLINKS) $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(CHECK_J) --protocol=30 +# Whole-suite gcov coverage report (HTML, with branch + decision coverage). +# Requires a build configured with --enable-coverage and the `gcovr` tool +# (pip install gcovr). Runs the suite in parallel (COVERAGE_J, default CHECK_J): +# this is safe because the coverage build uses -fprofile-update=atomic and +# libgcov locks the per-source .gcda during its merge, so concurrent rsync +# processes accumulate exactly (see COVERAGE_J above). Use COVERAGE_J=1 if your +# toolchain's libgcov does not lock .gcda merges. +.PHONY: coverage +coverage: all $(CHECK_PROGS) $(CHECK_SYMLINKS) + @case '$(CFLAGS)' in *--coverage*) ;; \ + *) echo "*** not a coverage build; reconfigure with --enable-coverage"; exit 1 ;; esac + @command -v gcovr >/dev/null 2>&1 || { echo "*** gcovr not found (pip install gcovr)"; exit 1; } + find . -name '*.gcda' -delete + @rc=0; $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(COVERAGE_J) || rc=$$?; \ + rm -rf coverage && mkdir -p coverage; \ + gcovr --root $(srcdir) --branches --decisions --print-summary \ + --html-details -o coverage/index.html . || exit $$?; \ + echo "Coverage report written to coverage/index.html"; \ + if test $$rc != 0; then \ + echo "*** test suite FAILED (status $$rc) -- coverage report still written above"; \ + fi; \ + exit $$rc + wildtest.o: wildtest.c t_stub.o lib/wildmatch.c rsync.h config.h wildtest$(EXEEXT): wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ $(CC) $(CFLAGS) $(LDFLAGS) -o $@ wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ $(LIBS) diff --git a/cleanup.c b/cleanup.c index 0493fbbb1..7f1864ccb 100644 --- a/cleanup.c +++ b/cleanup.c @@ -269,8 +269,16 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) break; } - if (called_from_signal_handler) + if (called_from_signal_handler) { +#ifdef GCOV_COVERAGE + /* _exit() bypasses the gcov atexit flush; rsync's generator (and + * other processes) normally finish via the signal handler, so + * without this they would write no .gcda. Harmless otherwise. */ + extern void __gcov_dump(void); + __gcov_dump(); +#endif _exit(exit_code); + } exit(exit_code); } diff --git a/configure.ac b/configure.ac index 4062651df..4faab5fcb 100644 --- a/configure.ac +++ b/configure.ac @@ -82,6 +82,32 @@ if test x"$enable_profile" = x"yes"; then CFLAGS="$CFLAGS -pg" fi +dnl Coverage build (gcov) for `make coverage`. NOTE: --enable-profile above is +dnl gprof (-pg) and is NOT coverage. -O0 keeps branch coverage meaningful; +dnl -fprofile-update=atomic keeps the shared .gcda counters correct while the +dnl suite runs many rsync processes in parallel. +AC_ARG_ENABLE(coverage, + AS_HELP_STRING([--enable-coverage],[build with gcov instrumentation for `make coverage`])) +if test x"$enable_coverage" = x"yes"; then + CFLAGS="$CFLAGS --coverage -fprofile-update=atomic -O0" + CXXFLAGS="$CXXFLAGS --coverage -fprofile-update=atomic -O0" + LDFLAGS="$LDFLAGS --coverage" + AC_DEFINE([GCOV_COVERAGE], 1, + [Flush gcov counters at exit_cleanup: rsync's children exit via _exit(), which bypasses the gcov atexit handler, so without this no .gcda is written for the receiver/generator/daemon-worker processes.]) +fi + +dnl openat2(RESOLVE_BENEATH) is used on Linux 5.6+ for the secure resolver. +dnl --disable-openat2 forces the portable per-component O_NOFOLLOW fallback to +dnl run as the primary resolver on ordinary Linux, so that tier is exercised +dnl (and coverage-counted) without needing a pre-5.6 kernel. Behaviour-neutral +dnl by default (the knob only REMOVES a tier when explicitly disabled). +AC_ARG_ENABLE(openat2, + AS_HELP_STRING([--disable-openat2],[do not use Linux openat2(RESOLVE_BENEATH); force the portable resolver (for exercising the fallback tier)])) +if test x"$enable_openat2" != x"no"; then + AC_DEFINE([HAVE_OPENAT2], 1, + [Define to use Linux openat2(RESOLVE_BENEATH) in secure_relative_open where available.]) +fi + AC_MSG_CHECKING([if md2man can create manpages]) if test x"$ac_cv_path_PYTHON3" = x; then AC_MSG_RESULT(no - python3 not found) diff --git a/main.c b/main.c index 78f0b8331..c54fd79bc 100644 --- a/main.c +++ b/main.c @@ -1618,6 +1618,11 @@ static void sigusr2_handler(UNUSED(int val)) if (!am_server) output_summary(); close_all(); +#ifdef GCOV_COVERAGE + /* The receiver child is killed here via SIGUSR2 and exits with _exit(), + * bypassing the gcov atexit flush; without this it writes no .gcda. */ + { extern void __gcov_dump(void); __gcov_dump(); } +#endif if (got_xfer_error) _exit(RERR_PARTIAL); _exit(0); diff --git a/syscall.c b/syscall.c index e317bccc3..8579b075f 100644 --- a/syscall.c +++ b/syscall.c @@ -33,7 +33,7 @@ #include #endif -#ifdef __linux__ +#if defined(__linux__) && defined(HAVE_OPENAT2) #include #include #endif @@ -1691,7 +1691,7 @@ static int path_has_dotdot_component(const char *path) return 0; } -#ifdef __linux__ +#if defined(__linux__) && defined(HAVE_OPENAT2) static int secure_relative_open_linux(const char *basedir, const char *relpath, int flags, mode_t mode) { struct open_how how; @@ -1791,11 +1791,13 @@ int secure_relative_open(const char *basedir, const char *relpath, int flags, mo return -1; } -#ifdef __linux__ +#if defined(__linux__) && defined(HAVE_OPENAT2) { int fd = secure_relative_open_linux(basedir, relpath, flags, mode); /* ENOSYS = kernel < 5.6 doesn't have the syscall even though - * glibc/kernel-headers do; fall through to the portable path. */ + * glibc/kernel-headers do; fall through to the portable path. + * (Built unconditionally unless --disable-openat2, which forces + * the portable resolver below so that tier is exercised.) */ if (fd != -1 || errno != ENOSYS) return fd; } From b475d93979fbf07eb0d220b52dc064916c9c66b9 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 08:14:39 +1000 Subject: [PATCH 10/15] testsuite: add COVERAGE.md matrix and -u/--force coverage COVERAGE.md is the living checklist mapping every CLI option (~142) and daemon parameter (~54) to its test(s), with depth / cross-dir status and remaining gaps, so the path-resolution restructure can see exactly what is guarded. update_test.py closes two of the documented gaps: -u/--update (keep a newer destination, update an older one) and --force (replace a non-empty destination directory with a file), both at depth. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/COVERAGE.md | 206 +++++++++++++++++++++++++++++++++++++++ testsuite/update_test.py | 62 ++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 testsuite/COVERAGE.md create mode 100644 testsuite/update_test.py diff --git a/testsuite/COVERAGE.md b/testsuite/COVERAGE.md new file mode 100644 index 000000000..44094b63f --- /dev/null +++ b/testsuite/COVERAGE.md @@ -0,0 +1,206 @@ +# rsync option / daemon-parameter test coverage matrix + +Living checklist for the test-coverage effort that precedes the path-handling +restructure of rsync's path resolution. The restructure rewrites parent-directory +resolution for essentially every option, so the goal here is a regression net +that exercises each option **at directory depth** (≥3 levels) and, where the +option spans trees, **across directory boundaries**, asserting the *specific +property* the option controls — not just `dest == src`. + +How to read the columns: + +* **test(s)** — the `testsuite/*_test.py` that exercise the option. Tests added + by this effort are marked `*new*`. +* **depth** — Y = asserted on entries ≥3 levels deep; `~` = exercised only at/near + the tree root; `n/a` = not a path-resolution option. +* **x-dir** — Y = exercised with the relevant aux tree (temp/backup/dest/partial) + **outside** the main tree; `—` = not a cross-directory option. +* **gap** — what is still missing. + +Status legend: ✓ property asserted · `~` shallow / by an existing ported test · +✗ no coverage. + +--- + +## Command-line options + +### Recursion / structure +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| -a, --archive | (all) | Y | — | ✓ ubiquitous | +| -r, --recursive | hands, delete-deep*new* | Y | — | ✓ | +| -R, --relative | relative, relative-implied*new* | Y | — | ✓ implied-dir attrs at depth | +| --no-implied-dirs | relative-implied*new* | Y | — | ✓ (proto 30+; proto 29 rejects multi-component path) | +| --inc-recursive / --no-inc-recursive | hardlinks | Y | — | `~` exercised, not isolated | +| -d, --dirs | dirs*new* | Y | — | ✓ no-recurse top layer | +| --old-dirs / --old-d | — | — | — | ✗ | +| -m, --prune-empty-dirs | prune-empty-dirs*new* | Y | — | ✓ incl. filter-emptied chains | + +### Links +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| -l, --links | links*new*, symlink-ignore | Y | — | ✓ | +| -L, --copy-links | links*new* | Y | — | ✓ deref file+dir | +| -k, --copy-dirlinks | links*new* | Y | — | ✓ follow dir-symlink | +| -K, --keep-dirlinks | symlink-dirlink-basis | Y | — | ✓ #715; skips on no-RESOLVE_BENEATH / --disable-openat2 | +| -H, --hard-links | hardlinks, hardlinks-deep*new* | Y | Y | ✓ cross-directory hardlink | +| --copy-unsafe-links | unsafe-links | `~` | — | `~` | +| --safe-links | safe-links | `~` | — | `~` | +| --munge-links | (daemon-munge*new* covers the daemon param) | — | — | `~` client option not isolated; local mode is a near no-op | + +### Metadata / permissions / ownership +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| -p, --perms | metadata-depth*new* | Y | — | ✓ exact modes per entry | +| -E, --executability | executability | `~` | — | `~` | +| --chmod | metadata-depth*new*, chmod-option | Y | — | ✓ | +| -A, --acls | acls, acls-depth*new* | Y | — | ✓ | +| -X, --xattrs | xattrs, xattrs-depth*new* | Y | — | ✓ | +| -t, --times | metadata-depth*new* | Y | — | ✓ | +| -U, --atimes | atimes | `~` | — | `~` (same set path as -t, covered deep) | +| --open-noatime | open-noatime | `~` | — | `~` | +| -N, --crtimes | crtimes | `~` | — | `~` (skips without crtimes support) | +| -O, --omit-dir-times | omit-times*new* | Y | — | ✓ | +| -J, --omit-link-times | omit-times*new* | Y | — | ✓ | +| -o, --owner | chown, ownership-depth*new* | Y | — | ✓ uid map root-gated | +| -g, --group | chgrp, ownership-depth*new* | Y | — | ✓ group remap non-root | +| --super / --fake-super | chown, chown-fake | `~` | — | `~` | +| --numeric-ids | — | — | — | ✗ client; daemon `numeric ids` also ✗ | +| --usermap / --groupmap | ownership-depth*new* | Y | — | ✓ groupmap non-root; usermap root-gated | +| --chown | ownership-depth*new* | Y | — | ✓ group half | +| -D / --devices / --specials | devices, devices-fake | `~` | — | `~` root/device-gated | +| --copy-devices / --write-devices | — | — | — | ✗ device-gated | +| -S, --sparse | sparse*new* | Y | — | ✓ hole preserved at depth | + +### Delta / temp / backup / dest (highest restructure risk) +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| -T, --temp-dir | temp-dir*new*, chmod-temp-dir | Y | Y | ✓ cross-dir rename | +| --partial | partial*new* | Y | — | ✓ partial kept in dest file | +| --partial-dir | partial*new*, symlink-dirlink-basis | Y | Y | ✓ relative (in-tree) + absolute (outside). Absolute **delta** resume is broken on master — asserts only the cross-dir write | +| --delay-updates | delay-updates, delay-updates-deep*new* | Y | — | ✓ per-dir staging | +| --inplace | inplace*new*, alt-dest | Y | — | ✓ inode preserved | +| --append / --append-verify | append*new* | Y | — | ✓ verify split is proto 30+ | +| -b, --backup / --backup-dir / --suffix | backup, backup-deep*new* | Y | Y | ✓ | +| --compare-dest / --copy-dest / --link-dest | alt-dest, alt-dest-deep*new* | Y | Y | ✓ link=hardlink, copy=copy, compare=skip | +| -y, --fuzzy | fuzzy | `~` | — | `~` | +| -u, --update | update*new* | Y | — | ✓ keeps newer dest, updates older | +| -W, --whole-file | (used widely; --no-whole-file ubiquitous) | n/a | — | `~` | +| --mkpath | mkpath | `~` | — | `~` | +| -x, --one-file-system | — | — | — | ✗ (needs a mount boundary) | +| --preallocate / --fsync | — | — | — | ✗ | +| -B, --block-size | — | — | — | ✗ | +| --max-alloc | — | — | — | ✗ | + +### Filtering +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| -f, --filter / -F | filter-depth*new*, merge | Y | — | ✓ deep per-dir merge | +| --exclude / --include | filter-depth*new*, exclude, exclude-lsh | Y | — | ✓ | +| --exclude-from / --include-from | files-from-depth*new* | Y | — | ✓ | +| -C, --cvs-exclude | cvs-exclude*new* | Y | — | ✓ incl. deep .cvsignore | +| --files-from | files-from-depth*new* | Y | — | ✓ | +| -0, --from0 | files-from-depth*new* | Y | — | ✓ | +| --max-size / --min-size | size-filter*new* | Y | — | ✓ | +| --existing / --ignore-existing | delete-deep*new* | Y | — | ✓ | +| --ignore-missing-args / --delete-missing-args | — | — | — | ✗ | + +### Deletion +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| --delete / --del | delete, delete-deep*new* | Y | — | ✓ deep subtree | +| --delete-before/during/delay/after | delete-deep*new* | Y | — | ✓ all four agree | +| --delete-excluded | delete | `~` | — | `~` | +| --max-delete | delete-deep*new* | Y | — | ✓ caps deletions | +| --remove-source-files | delete | `~` | — | `~` | +| --force | update*new* | Y | — | ✓ replaces a non-empty dir with a file | +| --ignore-errors | — | — | — | ✗ (client; daemon `ignore errors` also ✗) | + +### Comparison / checksum / compression +| option | test(s) | depth | x-dir | notes / gap | +|---|---|---|---|---| +| -c, --checksum | compare*new* | Y | — | ✓ catches stealth change | +| -I, --ignore-times | compare*new* | Y | — | ✓ | +| --size-only | compare*new* | Y | — | ✓ | +| -@, --modify-window | compare*new* | Y | — | ✓ | +| --checksum-choice / --checksum-seed | compress-options*new* | Y | — | ✓ every advertised algo | +| -z, --compress | daemon-gzip-{up,down}load, daemon-refuse-compress | `~` | — | `~` | +| --compress-choice / --compress-level / --skip-compress | compress-options*new* | Y | — | ✓ | + +### Output / reporting (path-irrelevant — checked for output shape) +| option | test(s) | notes / gap | +|---|---|---| +| -i, --itemize-changes | output-options*new*, itemize | ✓ | +| -n, --dry-run | output-options*new* | ✓ | +| --stats | output-options*new* | ✓ | +| --out-format | output-options*new* | ✓ | +| --list-only | output-options*new* | ✓ | +| -q, --quiet | output-options*new* | ✓ | +| --progress / -P | output-options*new* | ✓ (--progress) | +| -h, --human-readable / -8, --8-bit-output | output-options*new* | ✓ smoke | +| --version / --help | output-options*new* | ✓ | +| --info / --debug / --stderr / --no-motd / --outbuf | — | ✗ | +| -M, --remote-option / --log-file / --log-file-format | — | ✗ (daemon `log file` covered) | + +### Batch / connection / misc +| option | test(s) | notes / gap | +|---|---|---| +| --write-batch / --only-write-batch / --read-batch | batch-mode | `~` | +| -e, --rsh / --rsync-path | ssh-basic, many | `~` | +| --protocol | check29 / check30 (whole suite) | ✓ | +| --address / --port | daemon tests under --use-tcp | `~` | +| --password-file | daemon-auth*new* | ✓ | +| --early-input / daemon `early exec` | — | ✗ | +| --sockopts / --blocking-io / --timeout / --contimeout | — | ✗ | +| -4/-6, --ipv4/--ipv6 | — | ✗ | +| --stop-after / --stop-at | — | ✗ | +| --bwlimit | partial*new* (used, not asserted) | `~` | +| --copy-as | — | ✗ root-gated | +| --iconv | — | ✗ | +| -s/--secluded-args, --old-args, --trust-sender | (default arg-protection exercised) | `~` | + +--- + +## Daemon (rsyncd.conf) parameters + +| parameter | test(s) | notes / gap | +|---|---|---| +| path | daemon-access*new*, all daemon tests | ✓ incl. deep sub-path | +| read only | daemon-access*new*, daemon | ✓ | +| write only | daemon-access*new* | ✓ | +| list | daemon-access*new*, daemon | ✓ hidden-but-usable | +| use chroot | sender-flist-symlink-leak, daemon-chroot-acl | `~` (no=most tests; yes needs root) | +| munge symlinks | daemon-munge*new* | ✓ /rsyncd-munged/ add+strip | +| exclude / include | daemon-filter*new*, daemon | ✓ exclude | +| filter / exclude from / include from | — | ✗ (exclude covers the mechanism) | +| incoming chmod | daemon-filter*new*, chmod-option | ✓ | +| outgoing chmod | daemon-filter*new* | ✓ | +| auth users / secrets file | daemon-auth*new* | ✓ accept/reject/unauth | +| strict modes | daemon-auth*new* | ✓ rejects world-readable secrets | +| refuse options | daemon-refuse*new*, daemon-refuse-compress | ✓ named/wildcard/allow-list | +| pre-xfer exec / post-xfer exec | daemon-exec*new* | ✓ env + abort | +| early exec | — | ✗ (needs --early-input) | +| hosts allow / hosts deny | daemon (allow), daemon-chroot-acl (deny) | `~` (needs --use-tcp for real peer) | +| reverse lookup / forward lookup | daemon-chroot-acl | `~` reverse only | +| log file / transfer logging / log format | daemon | `~` set, not asserted | +| max verbosity | daemon | `~` | +| comment | daemon, daemon-access*new* | ✓ | +| numeric ids | — | ✗ (hard to observe non-root) | +| fake super | chown-fake (client side) | ✗ as daemon param | +| timeout / max connections / lock file | — | ✗ (need --use-tcp + concurrency) | +| temp dir / open noatime / ignore errors / ignore nonreadable | — | ✗ | +| charset / name converter / dont compress | — | ✗ | +| uid / gid / daemon uid / daemon gid / daemon chroot | build_rsyncd_conf (uid/gid when root), daemon-chroot-acl | `~` root-gated | +| motd file / pid file / port / address / socket options / listen backlog / proxy protocol / syslog facility / syslog tag | — | ✗ (server-startup/connection params) | + +--- + +## Known gaps worth a future pass +* Connection/timeout params (`--timeout`, `--contimeout`, daemon `timeout`, + `max connections`) need a real socket + concurrency (run under `--use-tcp`). +* Root-only behaviours (`-o`/`--usermap` uid remap, real devices, `use chroot + = yes`, daemon uid/gid) skip as non-root; run the suite as root to cover. +* `--ignore-errors`, `-x/--one-file-system`, `--numeric-ids` have no dedicated + test yet (lower restructure risk). +* Absolute `--partial-dir` + delta resume is broken on master; the test asserts + only the cross-directory write there and completes with `--whole-file`. diff --git a/testsuite/update_test.py b/testsuite/update_test.py new file mode 100644 index 000000000..4cc10048f --- /dev/null +++ b/testsuite/update_test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Coverage of -u (--update) and --force at depth. + +-u skips any destination file that is newer than the source. --force lets rsync +delete a non-empty destination directory when it must be replaced by a +non-directory. Both decide a per-entry action on a name whose parent chain the +resolver restructure rewrites, so check them several levels deep. +""" + +import os + +from rsyncfns import ( + FROMDIR, TODIR, + assert_same, make_tree, makepath, rmtree, run_rsync, test_fail, +) + +src = FROMDIR +deep = os.path.join('d1', 'd2', 'd3', 'f3') + + +# --- -u keeps a newer destination file, updates an older one ---------------- +rmtree(src) +rmtree(TODIR) +make_tree(src, depth=3) +run_rsync('-a', f'{src}/', f'{TODIR}/') + +# Make the deep source newer in content, but the DEST copy newer in time. +(src / deep).write_text("new source content\n") +keep = "destination is newer - keep me\n" +(TODIR / deep).write_text(keep) +st = os.stat(src / deep) +os.utime(TODIR / deep, (st.st_atime, st.st_mtime + 100)) # dest mtime newer + +run_rsync('-a', '-u', f'{src}/', f'{TODIR}/') +if (TODIR / deep).read_text() != keep: + test_fail("-u overwrote a destination file that was newer than the source") + +# An older destination file IS updated under -u. +os.utime(TODIR / deep, (st.st_atime, st.st_mtime - 100)) # dest mtime older +run_rsync('-a', '-u', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='-u updated an older dest file') + +# --- --force replaces a non-empty dest directory with a file at depth ------- +rmtree(src) +rmtree(TODIR) +makepath(src / 'd1' / 'd2' / 'd3') +(src / deep).write_text("now a regular file\n") # src: d1/d2/d3/f3 = file +makepath(TODIR / 'd1' / 'd2' / 'd3' / 'f3') # dest: f3 = non-empty dir +(TODIR / 'd1' / 'd2' / 'd3' / 'f3' / 'occupant').write_text("blocker\n") + +# Without --force the non-empty directory can't be replaced. +proc = run_rsync('-a', f'{src}/', f'{TODIR}/', check=False) +if proc.returncode == 0 and (TODIR / deep).is_file(): + test_fail("a non-empty directory was replaced by a file without --force") + +# With --force the directory is removed and the file takes its place. +run_rsync('-a', '--force', f'{src}/', f'{TODIR}/') +if not (TODIR / deep).is_file(): + test_fail("--force did not replace the directory with the file at depth") +assert_same(TODIR / deep, src / deep, label='--force replacement content') + +print("update: -u keeps newer dest / updates older; --force replaces a dir at depth") From b59f10944dbb00629dfd5283af497d6f79ab7b9a Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 08:48:42 +1000 Subject: [PATCH 11/15] receiver: fix absolute --partial-dir delta resume (false verification) A delta (--no-whole-file) resume whose basis is an absolute --partial-dir looped forever on exit code 23 ("failed verification -- update put into partial-dir"), stranding the correct data in the partial-dir and never populating the destination. Cause: an absolute --partial-dir makes the basis path absolute, but the receiver opened it with secure_relative_open(NULL, fnamecmp, ...), which by design rejects an absolute relpath (EINVAL). The basis fd was then -1, so receive_data() mapped no basis and (because the matched-block sum_update() is guarded by "if (mapbuf)") computed the whole-file verification checksum over the literal data only -> a spurious mismatch every run. (The data itself was correct, since the in-place update leaves the matched basis bytes in place.) Under a non-chroot daemon the in-place write went through the same call and failed outright. Fix: add secure_basis_open(), which treats an operator-trusted absolute basis path as (trusted directory + confined leaf) -- the same way secure_relative_open already trusts an absolute basedir while keeping O_NOFOLLOW on the leaf -- and use it for both the basis read and the inplace-partial write. The strict "reject absolute relpath" contract of secure_relative_open is left intact. Defense-in-depth: receive_data() now treats a block-match token with no mapped basis as a protocol inconsistency (it can only arise from a basis that the generator opened but the receiver could not), failing cleanly instead of silently dropping those bytes from the verify checksum or the output. Co-Authored-By: Claude Opus 4.7 (1M context) --- receiver.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/receiver.c b/receiver.c index f49931bf1..7d429fe84 100644 --- a/receiver.c +++ b/receiver.c @@ -83,6 +83,44 @@ static int updating_basis_or_equiv; #define MAX_UNIQUE_NUMBER 999999 #define MAX_UNIQUE_LOOP 100 +/* Open a basis/output path that may legitimately be an operator-trusted + * ABSOLUTE path -- e.g. an absolute --partial-dir ("a directory reserved for + * partial-dir work") or --backup-dir. secure_relative_open() deliberately + * rejects an absolute relpath, so feeding it the whole absolute partialptr + * (with a NULL basedir) returns EINVAL: the basis fd is then -1, no basis is + * mapped, and receive_data() omits every matched block from the whole-file + * verification checksum -> a spurious "failed verification" that strands the + * (correct) data in the partial-dir forever. + * + * The operator's directory is trusted; only the leaf basename is peer-supplied. + * So when basedir is NULL and relpath is absolute, split it into its directory + * (trusted) and leaf and confine just the leaf -- exactly how secure_relative_ + * open already trusts an absolute basedir while O_NOFOLLOW-confining the leaf. + * Anything else is a straight pass-through that preserves the strict contract. */ +static int secure_basis_open(const char *basedir, const char *relpath, int flags, mode_t mode) +{ + if (!basedir && relpath && *relpath == '/') { + const char *slash = strrchr(relpath, '/'); + const char *leaf = slash + 1; + char dirbuf[MAXPATHLEN]; + const char *dir; + if (slash == relpath) + dir = "/"; + else { + size_t dlen = slash - relpath; + if (dlen >= sizeof dirbuf) { + errno = ENAMETOOLONG; + return -1; + } + memcpy(dirbuf, relpath, dlen); + dirbuf[dlen] = '\0'; + dir = dirbuf; + } + return secure_relative_open(dir, leaf, flags, mode); + } + return secure_relative_open(basedir, relpath, flags, mode); +} + /* get_tmpname() - create a tmp filename for a given filename * * If a tmpdir is defined, use that as the directory to put it in. Otherwise, @@ -364,6 +402,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, stats.matched_data += len; + /* A block match can only be honored if we actually mapped the + * basis. If we didn't (basis open failed), the sender should + * never have been told a basis existed -- treat it as a protocol + * inconsistency rather than silently omitting these bytes from + * the verification checksum (which yields a spurious failure) or + * leaving a hole in the output. */ + if (!mapbuf) { + rprintf(FERROR, "got a block match with no basis file for %s [%s]\n", + full_fname(fname), who_am_i()); + exit_cleanup(RERR_PROTOCOL); + } + if (DEBUG_GTE(DELTASUM, 3)) { rprintf(FINFO, "chunk[%d] of size %ld at %s offset=%s%s\n", @@ -793,8 +843,9 @@ int recv_files(int f_in, int f_out, char *local_name) fnamecmp = fname; } - /* open the file */ - fd1 = secure_relative_open(basedir, fnamecmp, O_RDONLY, 0); + /* open the file (secure_basis_open tolerates an operator-trusted + * absolute fnamecmp, e.g. an absolute --partial-dir basis) */ + fd1 = secure_basis_open(basedir, fnamecmp, O_RDONLY, 0); if (fd1 == -1 && protocol_version < 29) { if (fnamecmp != fname) { @@ -884,7 +935,7 @@ int recv_files(int f_in, int f_out, char *local_name) * attacker could switch a directory to a symlink between * path validation and file open. */ if (use_secure_symlinks) - fd2 = secure_relative_open(NULL, fnametmp, O_WRONLY|O_CREAT, 0600); + fd2 = secure_basis_open(NULL, fnametmp, O_WRONLY|O_CREAT, 0600); else fd2 = do_open(fnametmp, O_WRONLY|O_CREAT, 0600); #ifdef linux From 3f4e43fbda8dfb60b90203667d00e522c7b51630 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 08:48:42 +1000 Subject: [PATCH 12/15] testsuite: assert absolute --partial-dir delta resume now works partial_test.py sub-test 5 deterministically asserts a delta (--no-whole-file) resume from an absolute, outside-tree --partial-dir reproduces the source and consumes the basis -- the regression guard for the receiver fix. Sub-test 4 keeps asserting the cross-directory partial WRITE on interrupt. Drop the --whole-file workaround and the 'broken on master' notes in the docstring and COVERAGE.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- testsuite/COVERAGE.md | 4 +--- testsuite/partial_test.py | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/testsuite/COVERAGE.md b/testsuite/COVERAGE.md index 44094b63f..6f6e37cf3 100644 --- a/testsuite/COVERAGE.md +++ b/testsuite/COVERAGE.md @@ -77,7 +77,7 @@ Status legend: ✓ property asserted · `~` shallow / by an existing ported test |---|---|---|---|---| | -T, --temp-dir | temp-dir*new*, chmod-temp-dir | Y | Y | ✓ cross-dir rename | | --partial | partial*new* | Y | — | ✓ partial kept in dest file | -| --partial-dir | partial*new*, symlink-dirlink-basis | Y | Y | ✓ relative (in-tree) + absolute (outside). Absolute **delta** resume is broken on master — asserts only the cross-dir write | +| --partial-dir | partial*new*, symlink-dirlink-basis | Y | Y | ✓ relative (in-tree) + absolute (outside), incl. delta resume from an absolute outside-tree partial | | --delay-updates | delay-updates, delay-updates-deep*new* | Y | — | ✓ per-dir staging | | --inplace | inplace*new*, alt-dest | Y | — | ✓ inode preserved | | --append / --append-verify | append*new* | Y | — | ✓ verify split is proto 30+ | @@ -202,5 +202,3 @@ Status legend: ✓ property asserted · `~` shallow / by an existing ported test = yes`, daemon uid/gid) skip as non-root; run the suite as root to cover. * `--ignore-errors`, `-x/--one-file-system`, `--numeric-ids` have no dedicated test yet (lower restructure risk). -* Absolute `--partial-dir` + delta resume is broken on master; the test asserts - only the cross-directory write there and completes with `--whole-file`. diff --git a/testsuite/partial_test.py b/testsuite/partial_test.py index 97b6f69af..1c8e9531a 100644 --- a/testsuite/partial_test.py +++ b/testsuite/partial_test.py @@ -9,11 +9,6 @@ basename. All of this is parent- and cross-directory path resolution -- what the resolver restructure rewrites -- so exercise it on a file several levels deep, with the absolute partial-dir kept OUTSIDE the destination tree. - -Note: a *delta* resume from an absolute partial-dir currently fails whole-file -verification on master (it re-puts the partial and never converges). This test -therefore only asserts the cross-directory WRITE of the partial for that case -and completes it with --whole-file, which is the clearly-correct baseline. """ import os @@ -115,7 +110,7 @@ def interrupt_transfer(extra_args, partial_path): f'{src}/', f'{TODIR}/') assert_same(TODIR / deep, src / deep, label='rel partial-dir resume') -# --- 4. absolute --partial-dir OUTSIDE the tree (cross-dir): interrupt ----- +# --- 4. absolute --partial-dir OUTSIDE the tree (cross-dir): interrupt write - ext = SCRATCHDIR / 'partials' # sibling of from/ and to/ -- outside both rmtree(ext) ext.mkdir() @@ -124,7 +119,21 @@ def interrupt_transfer(extra_args, partial_path): if not (ext / 'f3').is_file() or not is_prefix(ext / 'f3'): test_fail("absolute --partial-dir did not write the partial to the " "outside-tree dir") -run_rsync('-a', f'--partial-dir={ext}', '--whole-file', f'{src}/', f'{TODIR}/') -assert_same(TODIR / deep, src / deep, label='abs partial-dir resume') + +# --- 5. absolute --partial-dir delta resume completes (regression guard) ---- +# A delta (--no-whole-file) resume from an absolute, outside-tree partial-dir +# used to fail whole-file verification forever: the receiver couldn't open the +# absolute basis, so matched blocks were dropped from the verify checksum. +rmtree(src) +rmtree(TODIR) +rmtree(ext) +makepath(src / deepdir, ext) +make_data_file(src / deep, 1_000_000) +(ext / 'f3').write_bytes((src / deep).read_bytes()[:400_000]) # clean prefix +run_rsync('-a', f'--partial-dir={ext}', '--no-whole-file', f'{src}/', f'{TODIR}/') +assert_same(TODIR / deep, src / deep, label='abs partial-dir delta resume') +if (ext / 'f3').exists(): + test_fail("absolute --partial-dir basis was not consumed after a " + "successful delta resume") print("partial: --partial + relative/absolute --partial-dir verified at depth") From 4df6dae2ac1af06c78c861f90b5965ff9815b1e0 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 09:23:54 +1000 Subject: [PATCH 13/15] build: add 'make coverage-tcp' and drop deprecated gcovr --branches coverage-tcp reuses the coverage recipe with --use-tcp (daemon tests over a real loopback rsyncd, which also runs the require_tcp-only tests) and a separate report directory, via COVERAGE_RUNFLAGS / COVERAGE_DIR. Verified end to end: pipe run reports 63.9% lines, the TCP run 64.5% (it exercises more code). Also drop gcovr's --branches flag: it is deprecated in gcovr 8 and branch + decision coverage still appear in --print-summary and the HTML without it. Co-Authored-By: Claude Opus 4.7 (1M context) --- Makefile.in | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8f3b04c9d..9e9b9a82e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -281,7 +281,7 @@ clean: cleantests git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 @MAKE_RRSYNC_1@ \ *.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp rm -f *.gcno *.gcda lib/*.gcno lib/*.gcda zlib/*.gcno zlib/*.gcda popt/*.gcno popt/*.gcda - rm -rf coverage + rm -rf coverage coverage-tcp .PHONY: cleantests cleantests: @@ -335,6 +335,13 @@ CHECK_J = 8 # `make coverage COVERAGE_J=1` if your libgcov does not lock .gcda merges. COVERAGE_J = $(CHECK_J) +# Output directory and extra runtests.py flags for `make coverage`. The +# `coverage-tcp` target reuses the coverage recipe with --use-tcp (real +# loopback rsyncd, which exercises the TCP accept/auth path and the +# require_tcp-only tests) and a separate output directory. +COVERAGE_DIR = coverage +COVERAGE_RUNFLAGS = + .PHONY: check check: all $(CHECK_PROGS) $(CHECK_SYMLINKS) $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(CHECK_J) @@ -360,16 +367,22 @@ coverage: all $(CHECK_PROGS) $(CHECK_SYMLINKS) *) echo "*** not a coverage build; reconfigure with --enable-coverage"; exit 1 ;; esac @command -v gcovr >/dev/null 2>&1 || { echo "*** gcovr not found (pip install gcovr)"; exit 1; } find . -name '*.gcda' -delete - @rc=0; $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(COVERAGE_J) || rc=$$?; \ - rm -rf coverage && mkdir -p coverage; \ - gcovr --root $(srcdir) --branches --decisions --print-summary \ - --html-details -o coverage/index.html . || exit $$?; \ - echo "Coverage report written to coverage/index.html"; \ + @rc=0; $(srcdir)/runtests.py --rsync-bin=`pwd`/rsync$(EXEEXT) -j $(COVERAGE_J) $(COVERAGE_RUNFLAGS) || rc=$$?; \ + rm -rf $(COVERAGE_DIR) && mkdir -p $(COVERAGE_DIR); \ + gcovr --root $(srcdir) --decisions --print-summary \ + --html-details -o $(COVERAGE_DIR)/index.html . || exit $$?; \ + echo "Coverage report written to $(COVERAGE_DIR)/index.html"; \ if test $$rc != 0; then \ echo "*** test suite FAILED (status $$rc) -- coverage report still written above"; \ fi; \ exit $$rc +# Same as `make coverage` but with the daemon tests run over a real loopback +# rsyncd (--use-tcp), into a separate report directory. +.PHONY: coverage-tcp +coverage-tcp: + $(MAKE) coverage COVERAGE_RUNFLAGS=--use-tcp COVERAGE_DIR=coverage-tcp + wildtest.o: wildtest.c t_stub.o lib/wildmatch.c rsync.h config.h wildtest$(EXEEXT): wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ $(CC) $(CFLAGS) $(LDFLAGS) -o $@ wildtest.o lib/compat.o lib/snprintf.o @BUILD_POPT@ $(LIBS) From f81fd4aad383909a611c52b4c9948a2373e8a081 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 09:23:54 +1000 Subject: [PATCH 14/15] ci: add an Ubuntu gcov coverage job Builds with --enable-coverage and runs the suite under both transports (make coverage, then make coverage-tcp). gcovr's line/branch/decision totals are printed to the step log and also written to the GitHub step summary, so the coverage numbers are visible directly in the CI output; the HTML reports are uploaded as an artifact. make coverage exits with the suite's status, so a test regression fails the job. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/coverage.yml | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..71d9f37fd --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,71 @@ +name: Coverage (Ubuntu) + +on: + push: + branches: [ master ] + paths-ignore: + - '.github/workflows/*.yml' + - '!.github/workflows/coverage.yml' + pull_request: + branches: [ master ] + paths-ignore: + - '.github/workflows/*.yml' + - '!.github/workflows/coverage.yml' + schedule: + - cron: '42 9 * * *' + workflow_dispatch: + +jobs: + coverage: + runs-on: ubuntu-latest + name: gcov coverage + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: prep + run: | + sudo apt-get update + sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl gcovr + echo "/usr/local/bin" >>$GITHUB_PATH + - name: configure + run: ./configure --enable-coverage --with-rrsync + - name: make + run: make + - name: info + run: rsync --version + # Two coverage runs: the default pipe transport, then a second pass over a + # real loopback rsyncd (--use-tcp) which also exercises the require_tcp-only + # tests. gcovr's --print-summary line/branch/decision totals go to the step + # log (and the job summary below), so the numbers are visible in CI. + # `make coverage` exits with the suite's status, so a regression fails CI. + - name: coverage (pipe transport) + run: | + set -o pipefail + sudo make coverage 2>&1 | tee cov-pipe.log + - name: coverage (TCP transport) + run: | + set -o pipefail + sudo make coverage-tcp 2>&1 | tee cov-tcp.log + - name: coverage summary + if: always() + run: | + { + echo "## gcov coverage" + echo "### Pipe transport (\`make coverage\`)" + echo '```' + grep -E '^(lines|functions|branches|decisions):' cov-pipe.log || echo '(no summary -- see step log)' + echo '```' + echo "### TCP transport (\`make coverage-tcp\`)" + echo '```' + grep -E '^(lines|functions|branches|decisions):' cov-tcp.log || echo '(no summary -- see step log)' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: upload HTML reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-html + path: | + coverage + coverage-tcp From 659047b737cebb311fee2de472c8a988215586e9 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sun, 24 May 2026 09:44:49 +1000 Subject: [PATCH 15/15] ci: declare new metadata-coverage test skips for macOS and Cygwin acls-depth skips where ACLs/setfacl are unavailable (macOS, Cygwin) like the existing acls tests, and sparse skips on APFS (macOS), where a seek-written hole isn't allocated sparsely. Add them to the per-platform RSYNC_EXPECT_SKIPPED lists so the skip-set assertion stays accurate. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/cygwin-build.yml | 2 +- .github/workflows/macos-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cygwin-build.yml b/.github/workflows/cygwin-build.yml index 946166fa7..7433b8396 100644 --- a/.github/workflows/cygwin-build.yml +++ b/.github/workflows/cygwin-build.yml @@ -44,7 +44,7 @@ jobs: # package installed above), verified on a real Cygwin host. The real # chown/devices tests still skip (need root/mknod), as do the # RESOLVE_BENEATH symlink-race tests. - run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,sender-flist-symlink-leak,simd-checksum,symlink-dirlink-basis make check' + run: bash -c 'RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,acls,bare-do-open-symlink-race,chdir-symlink-race,chown,daemon-chroot-acl,devices,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,sender-flist-symlink-leak,simd-checksum,symlink-dirlink-basis make check' - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport. diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index f019e6c45..e73de3ed7 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -44,7 +44,7 @@ jobs: # chown-fake / devices-fake / xattrs / xattrs-hlink now RUN on macOS # (rsyncfns.py drives xattrs via the `xattr` command), verified on a # real macOS host, so they're no longer in the skip set. - run: sudo RSYNC_EXPECT_SKIPPED=acls-default,chmod-temp-dir,daemon-chroot-acl,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,simd-checksum make check + run: sudo RSYNC_EXPECT_SKIPPED=acls-default,acls-depth,chmod-temp-dir,daemon-chroot-acl,dir-sgid,open-noatime,protected-regular,proxy-response-line-too-long,simd-checksum,sparse make check - name: check (TCP daemon transport) # Second run with daemon tests over a real loopback rsyncd; the default # 'make check' above uses the secure stdio-pipe transport.