Skip to content

fix(manifest): preserve 64-bit list metadata ids #1334

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fallintoplace wants to merge 2 commits into
base: main
Choose a base branch
from
+105 −9
Open

fix(manifest): preserve 64-bit list metadata ids #1334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
61c0b21
fix(manifest): preserve 64-bit list metadata ids
fallintoplace Jun 27, 2026
46d5a04
test(manifest): cover max int64 list metadata
fallintoplace Jun 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions manifest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1427,12 +1427,12 @@ func NewManifestListWriterV1(out io.Writer, snapshotID int64, parentSnapshot *in

parentSnapshotStr := "null"
if parentSnapshot != nil {
parentSnapshotStr = strconv.Itoa(int(*parentSnapshot))
parentSnapshotStr = strconv.FormatInt(*parentSnapshot, 10)
}

return m, m.init(map[string][]byte{
"format-version": []byte(strconv.Itoa(m.version)),
"snapshot-id": []byte(strconv.Itoa(int(snapshotID))),
"snapshot-id": []byte(strconv.FormatInt(snapshotID, 10)),
"parent-snapshot-id": []byte(parentSnapshotStr),
})
}
Expand All @@ -1447,13 +1447,13 @@ func NewManifestListWriterV2(out io.Writer, snapshotID, sequenceNumber int64, pa

parentSnapshotStr := "null"
if parentSnapshot != nil {
parentSnapshotStr = strconv.Itoa(int(*parentSnapshot))
parentSnapshotStr = strconv.FormatInt(*parentSnapshot, 10)
}

return m, m.init(map[string][]byte{
"format-version": []byte(strconv.Itoa(m.version)),
"snapshot-id": []byte(strconv.Itoa(int(snapshotID))),
"sequence-number": []byte(strconv.Itoa(int(sequenceNumber))),
"snapshot-id": []byte(strconv.FormatInt(snapshotID, 10)),
"sequence-number": []byte(strconv.FormatInt(sequenceNumber, 10)),
"parent-snapshot-id": []byte(parentSnapshotStr),
})
}
Expand All @@ -1468,14 +1468,14 @@ func NewManifestListWriterV3(out io.Writer, snapshotId, sequenceNumber, firstRow
}
parentSnapshotStr := "null"
if parentSnapshot != nil {
parentSnapshotStr = strconv.Itoa(int(*parentSnapshot))
parentSnapshotStr = strconv.FormatInt(*parentSnapshot, 10)
}

return m, m.init(map[string][]byte{
"format-version": []byte(strconv.Itoa(m.version)),

@tanmayrauth tanmayrauth Jun 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice that you caught first-row-id here too, not just the fields shared with v1/v2, FormatInt is the right call for all of these since they're all 64-bit per the spec.

"snapshot-id": []byte(strconv.Itoa(int(snapshotId))),
"sequence-number": []byte(strconv.Itoa(int(sequenceNumber))),
"first-row-id": []byte(strconv.Itoa(int(firstRowID))),
"snapshot-id": []byte(strconv.FormatInt(snapshotId, 10)),
"sequence-number": []byte(strconv.FormatInt(sequenceNumber, 10)),
"first-row-id": []byte(strconv.FormatInt(firstRowID, 10)),
"parent-snapshot-id": []byte(parentSnapshotStr),
})
}
Expand Down
96 changes: 96 additions & 0 deletions manifest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"errors"
"io"
"io/fs"
"math"
"strconv"
"testing"
"time"
Expand Down Expand Up @@ -2023,6 +2024,101 @@ func (m *ManifestTestSuite) TestV3ManifestListWriterPersistsPerManifestFirstRowI
m.EqualValues(5022, *writer.NextRowID())
}

func (m *ManifestTestSuite) TestManifestListWriterMetadataPreservesInt64Values() {

@tanmayrauth tanmayrauth Jun 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1<<40 already proves the >2^32 case well. Might be worth also asserting a value up near math.MaxInt64 (a realistic snapshot id like 9223372036854775807), since that's the boundary a future regression to a
32-bit path would corrupt first.

parentSnapshot := int64(1 << 40)

tests := []struct {
name string
build func(io.Writer) (*ManifestListWriter, error)
expected map[string]string
}{
{
name: "v1",
build: func(w io.Writer) (*ManifestListWriter, error) {
return NewManifestListWriterV1(w, (1<<40)+1, &parentSnapshot)
},
expected: map[string]string{
"format-version": "1",
"snapshot-id": "1099511627777",
"parent-snapshot-id": "1099511627776",
},
},
{
name: "v2",
build: func(w io.Writer) (*ManifestListWriter, error) {
return NewManifestListWriterV2(w, (1<<40)+2, (1<<40)+3, &parentSnapshot)
},
expected: map[string]string{
"format-version": "2",
"snapshot-id": "1099511627778",
"sequence-number": "1099511627779",
"parent-snapshot-id": "1099511627776",
},
},
{
name: "v3",
build: func(w io.Writer) (*ManifestListWriter, error) {
return NewManifestListWriterV3(w, (1<<40)+4, (1<<40)+5, (1<<40)+6, &parentSnapshot)
},
expected: map[string]string{
"format-version": "3",
"snapshot-id": "1099511627780",
"sequence-number": "1099511627781",
"first-row-id": "1099511627782",
"parent-snapshot-id": "1099511627776",
},
},
}

for _, tt := range tests {
m.Run(tt.name, func() {
var buf bytes.Buffer
writer, err := tt.build(&buf)
m.Require().NoError(err)
m.Require().NoError(writer.Close())

reader, err := ocf.NewReader(&buf)
m.Require().NoError(err)
defer func() {
m.Require().NoError(reader.Close())
}()

meta := reader.Metadata()
for key, want := range tt.expected {
m.Equal(want, string(meta[key]))
}
})
}

m.Run("v3_near_max_int64", func() {
maxParentSnapshot := int64(math.MaxInt64 - 3)

var buf bytes.Buffer
writer, err := NewManifestListWriterV3(
&buf,
math.MaxInt64-2,
math.MaxInt64-1,
math.MaxInt64,
&maxParentSnapshot,
)
m.Require().NoError(err)
m.Require().NoError(writer.Close())

reader, err := ocf.NewReader(&buf)
m.Require().NoError(err)
defer func() {
m.Require().NoError(reader.Close())
}()

meta := reader.Metadata()
m.Equal("3", string(meta["format-version"]))
m.Equal(strconv.FormatInt(math.MaxInt64-2, 10), string(meta["snapshot-id"]))
m.Equal(strconv.FormatInt(math.MaxInt64-1, 10), string(meta["sequence-number"]))
m.Equal(strconv.FormatInt(math.MaxInt64, 10), string(meta["first-row-id"]))
m.Equal(strconv.FormatInt(maxParentSnapshot, 10), string(meta["parent-snapshot-id"]))
})
}

func (m *ManifestTestSuite) TestV3PrepareEntrySequenceNumberValidation() {
// Test v3writerImpl.prepareEntry sequence number validation logic
v3Writer := v3writerImpl{}
Expand Down
Loading