[3.13] gh-83869: tarfile: compute next header offset using pax size for sparse file (GH-18562) (GH-155868)

In case of a sparse file, the tarinfo.size attribute is set to the sparse
file expanded size (pax attribute GNU.sparse.size or GNU.sparse.size) and
do not correspond to the actual size of the data block. The size of the
data block is specified by the size pax header if present or by the ustar
size header.
Moreover, for GNU sparse 1.0 files, the data block start at the beginning of
the sparse mapping and not after the sparse mapping and so the offset should
be computed from here.
(cherry picked from commit 1a52eae)

Co-authored-by: Valentin Samir <valentin.samir@crans.org>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

Author: 3 people

Lib/tarfile.py

@@ -1605,17 +1605,22 @@ def _proc_pax(self, tarfile):
         if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
             # Patch the TarInfo object with the extended header info.
             next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
-            next.offset = self.offset
 
             if "size" in pax_headers:
                 # If the extended header replaces the size field,
                 # we need to recalculate the offset where the next
                 # header starts.
-                offset = next.offset_data
+                offset = next.offset + BLOCKSIZE
                 if next.isreg() or next.type not in SUPPORTED_TYPES:
-                    offset += next._block(next.size)
+                    try:
+                        size = PAX_NUMBER_FIELDS["size"](pax_headers["size"])
+                    except ValueError:
+                        size = 0
+                    offset += next._block(size)
                 tarfile.offset = offset
 
+            next.offset = self.offset
+
         return next
 
     def _proc_gnusparse_00(self, next, raw_headers):

Lib/test/test_tarfile.py

@@ -1348,6 +1348,37 @@ def test_sparse_file_01(self):
     def test_sparse_file_10(self):
         self._test_sparse_file("gnu/sparse-1.0")
 
+    def test_sparse_file_10_pax_size(self):
+        # gh-83869: when the pax header replaces the size field, the offset
+        # of the next header must be computed from the size of the data in
+        # the archive, not from the apparent size of the sparse file.
+        data = b"payload!" * 4
+        realsize = 1 << 20
+        smap = b"1\n%d\n%d\n" % (realsize - len(data), len(data))
+        smap += b"\0" * (-len(smap) % tarfile.BLOCKSIZE)
+
+        sparse = tarfile.TarInfo("sparse")
+        sparse.size = len(smap) + len(data)
+        sparse.pax_headers = {
+            "GNU.sparse.major": "1",
+            "GNU.sparse.minor": "0",
+            "GNU.sparse.name": "sparse",
+            "GNU.sparse.realsize": str(realsize),
+            "size": str(sparse.size),
+        }
+        buf = sparse.tobuf(tarfile.PAX_FORMAT)
+        buf += smap + data + b"\0" * (-len(data) % tarfile.BLOCKSIZE)
+
+        last = tarfile.TarInfo("last")
+        last.size = len(data)
+        buf += last.tobuf(tarfile.PAX_FORMAT)
+        buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE)
+        buf += b"\0" * (tarfile.BLOCKSIZE * 2)
+
+        with tarfile.open(fileobj=io.BytesIO(buf)) as tar:
+            self.assertEqual(tar.getnames(), ["sparse", "last"])
+            self.assertEqual(tar.extractfile("last").read(), data)
+
     @staticmethod
     def _fs_supports_holes():
         # Return True if the platform knows the st_blocks stat attribute and
@@ -1400,6 +1431,31 @@ def test_pax_global_headers(self):
         finally:
             tar.close()
 
+    def test_offset_after_global_header(self):
+        # gh-83869: a global header is a member of its own, the member which
+        # follows it keeps the offset of its own header.
+        rec = b"30 comment=global header here\n"
+        glob = tarfile.TarInfo("././@PaxHeader")
+        glob.type = tarfile.XGLTYPE
+        glob.size = len(rec)
+        buf = glob.tobuf(tarfile.USTAR_FORMAT)
+        buf += rec + b"\0" * (-len(rec) % tarfile.BLOCKSIZE)
+
+        member = tarfile.TarInfo("member")
+        data = b"hello\n"
+        member.size = len(data)
+        offset = len(buf)
+        buf += member.tobuf(tarfile.USTAR_FORMAT)
+        buf += data + b"\0" * (-len(data) % tarfile.BLOCKSIZE)
+        buf += b"\0" * (tarfile.BLOCKSIZE * 2)
+
+        with tarfile.open(fileobj=io.BytesIO(buf)) as tar:
+            tarinfo = tar.getmember("member")
+            self.assertEqual(tarinfo.offset, offset)
+            self.assertEqual(tarinfo.pax_headers.get("comment"),
+                             "global header here")
+            self.assertEqual(tar.extractfile(tarinfo).read(), data)
+
     def test_pax_number_fields(self):
         # All following number fields are read from the pax header.
         tar = tarfile.open(tarname, encoding="iso8859-1")

Misc/NEWS.d/next/Library/2020-02-19-16-35-52.gh-issue-83869.EPD_zn.rst

@@ -0,0 +1,6 @@
+Fix :mod:`tarfile` reading an archive with a GNU sparse 1.0 member whose
+size is set in the pax extended header.
+The offset of the next header was computed from the offset of the data,
+which is already past the sparse map, and from the size of the member,
+which can be the apparent size of the sparse file.
+All following members were unreachable.