From 1a6c642a0d741995735e43ddb468602b53d14a02 Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 17:21:43 +0200 Subject: [PATCH 01/21] Implement high-performance sorted writing support in IcebergIO --- sdks/java/io/iceberg/build.gradle | 1 + .../beam/sdk/io/iceberg/IcebergRowSorter.java | 234 ++++++++++++++++++ .../io/iceberg/WriteGroupedRowsToFiles.java | 16 +- .../iceberg/WritePartitionedRowsToFiles.java | 5 +- .../io/iceberg/WriteUngroupedRowsToFiles.java | 14 +- .../sdk/io/iceberg/IcebergIOWriteTest.java | 88 +++++++ .../sdk/io/iceberg/IcebergRowSorterTest.java | 229 +++++++++++++++++ 7 files changed, 581 insertions(+), 6 deletions(-) create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index bbd55fee2fc8..141de355813c 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -46,6 +46,7 @@ dependencies { implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":model:pipeline", configuration: "shadow") + implementation project(path: ":sdks:java:extensions:sorter") implementation library.java.avro implementation library.java.slf4j_api implementation library.java.joda_time diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java new file mode 100644 index 000000000000..96e35a9cdcce --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Date; +import java.util.Iterator; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.extensions.sorter.BufferedExternalSorter; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.Row; +import org.apache.iceberg.NullOrder; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortDirection; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.ReadableInstant; + +/** + * A utility class to sort Beam {@link Row}s based on an Iceberg {@link SortOrder}. Leverages {@link + * BufferedExternalSorter} to spill to local disk when elements exceed memory limit. + */ +class IcebergRowSorter implements Serializable { + + public static Iterable sortRows( + Iterable rows, + SortOrder sortOrder, + Schema icebergSchema, + org.apache.beam.sdk.schemas.Schema beamSchema) { + + if (sortOrder == null || !sortOrder.isSorted()) { + return rows; + } + + BufferedExternalSorter.Options sorterOptions = BufferedExternalSorter.options(); + BufferedExternalSorter sorter = BufferedExternalSorter.create(sorterOptions); + RowCoder rowCoder = RowCoder.of(beamSchema); + + try { + for (Row row : rows) { + byte[] keyBytes = encodeSortKey(row, sortOrder, icebergSchema, beamSchema); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + rowCoder.encode(row, baos); + byte[] valBytes = baos.toByteArray(); + sorter.add(KV.of(keyBytes, valBytes)); + } + + Iterable> sortedKVs = sorter.sort(); + return new Iterable() { + @Override + public Iterator iterator() { + final Iterator> it = sortedKVs.iterator(); + return new Iterator() { + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public Row next() { + KV next = it.next(); + try { + ByteArrayInputStream bais = new ByteArrayInputStream(next.getValue()); + return rowCoder.decode(bais); + } catch (IOException e) { + throw new RuntimeException("Failed to decode Row during sorting", e); + } + } + }; + } + }; + + } catch (IOException e) { + throw new RuntimeException("Failed to sort rows with external sorter", e); + } + } + + @SuppressWarnings("nullness") + public static byte[] encodeSortKey( + Row row, + SortOrder sortOrder, + Schema icebergSchema, + org.apache.beam.sdk.schemas.Schema beamSchema) + throws IOException { + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (SortField field : sortOrder.fields()) { + String colName = icebergSchema.findColumnName(field.sourceId()); + Object val = row.getValue(colName); + + if (!field.transform().isIdentity()) { + Object icebergVal = + IcebergUtils.beamRowToIcebergRecord(icebergSchema, row).getField(colName); + if (icebergVal != null) { + val = field.transform().apply(icebergVal); + } else { + val = null; + } + } + + boolean isNull = (val == null); + boolean isDesc = (field.direction() == SortDirection.DESC); + boolean nullsFirst = (field.nullOrder() == NullOrder.NULLS_FIRST); + + // Determine correct header prefix to fulfill the NullOrder contracts + byte prefixByte; + if (isNull) { + if (isDesc) { + // Descending: High byte keys sort first. + // If Nulls First -> Null gets highest byte (0xFF) + // If Nulls Last -> Null gets lowest byte (0x00) + prefixByte = nullsFirst ? (byte) 0xFF : (byte) 0x00; + } else { + // Ascending: Low byte keys sort first. + // If Nulls First -> Null gets lowest byte (0x00) + // If Nulls Last -> Null gets highest byte (0xFF) + prefixByte = nullsFirst ? (byte) 0x00 : (byte) 0xFF; + } + } else { + if (isDesc) { + // If non-null and Descending, use a neutral value that sits opposite to the null byte + prefixByte = nullsFirst ? (byte) 0xFE : (byte) 0x01; + } else { + prefixByte = nullsFirst ? (byte) 0x01 : (byte) 0x00; + } + } + + baos.write(prefixByte); + + if (!isNull) { + byte[] valBytes = encodeValue(val); + // Bitwise invert non-null bytes to sort descending lexicographically + if (isDesc) { + for (int i = 0; i < valBytes.length; i++) { + valBytes[i] = (byte) ~valBytes[i]; + } + } + baos.write(valBytes); + } + } + + return baos.toByteArray(); + } + + @SuppressWarnings("JavaUtilDate") + private static byte[] encodeValue(@Nullable Object val) throws IOException { + if (val == null) { + return new byte[0]; + } + if (val instanceof String) { + return encodeString((String) val); + } else if (val instanceof Integer) { + int v = (Integer) val; + return ByteBuffer.allocate(4).putInt(v ^ Integer.MIN_VALUE).array(); + } else if (val instanceof Long) { + long v = (Long) val; + return ByteBuffer.allocate(8).putLong(v ^ Long.MIN_VALUE).array(); + } else if (val instanceof Float) { + int bits = Float.floatToIntBits((Float) val); + bits = (bits >= 0) ? (bits ^ Integer.MIN_VALUE) : ~bits; + return ByteBuffer.allocate(4).putInt(bits).array(); + } else if (val instanceof Double) { + long bits = Double.doubleToLongBits((Double) val); + bits = (bits >= 0) ? (bits ^ Long.MIN_VALUE) : ~bits; + return ByteBuffer.allocate(8).putLong(bits).array(); + } else if (val instanceof Boolean) { + return new byte[] {((Boolean) val) ? (byte) 0x01 : (byte) 0x00}; + } else if (val instanceof byte[]) { + return encodeByteArray((byte[]) val); + } else if (val instanceof ByteBuffer) { + return encodeByteArray(((ByteBuffer) val).array()); + } else if (val instanceof ReadableInstant) { + long enc = ((ReadableInstant) val).getMillis() ^ Long.MIN_VALUE; + return ByteBuffer.allocate(8).putLong(enc).array(); + } else if (val instanceof Instant) { + long enc = ((Instant) val).toEpochMilli() ^ Long.MIN_VALUE; + return ByteBuffer.allocate(8).putLong(enc).array(); + } else if (val instanceof Date) { + long enc = ((Date) val).getTime() ^ Long.MIN_VALUE; + return ByteBuffer.allocate(8).putLong(enc).array(); + } + + return encodeString(val.toString()); + } + + private static byte[] encodeString(String s) throws IOException { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + return encodeByteArray(bytes); + } + + /** + * Escape protocol to cleanly prevent collisions. Maps 0x00 -> [0x01, 0x01] Maps 0x01 -> [0x01, + * 0x02] Safely terminates sequence with 0x00. + */ + private static byte[] encodeByteArray(byte[] bytes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(bytes.length + 2); + for (byte b : bytes) { + if (b == 0x00) { + baos.write(0x01); + baos.write(0x01); + } else if (b == 0x01) { + baos.write(0x01); + baos.write(0x02); + } else { + baos.write(b); + } + } + baos.write(0x00); // Safe boundary delimiter + return baos.toByteArray(); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java index 12d9570d4a38..dde519128f63 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java @@ -17,7 +17,9 @@ */ package org.apache.beam.sdk.io.iceberg; +import java.util.Iterator; import java.util.List; +import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; @@ -30,6 +32,7 @@ import org.apache.beam.sdk.values.WindowedValue; import org.apache.beam.sdk.values.WindowedValues; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Catalog; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -101,11 +104,22 @@ public void processElement( IcebergDestination destination = dynamicDestinations.instantiateDestination(tableIdentifier); WindowedValue windowedDestination = WindowedValues.of(destination, window.maxTimestamp(), window, paneInfo); + Iterator rowIt = element.getValue().iterator(); + if (!rowIt.hasNext()) { + return; + } + Row firstRow = rowIt.next(); + Schema dataSchema = firstRow.getSchema(); + RecordWriterManager writer; try (RecordWriterManager openWriter = new RecordWriterManager(getCatalog(), filePrefix, maxFileSize, Integer.MAX_VALUE)) { writer = openWriter; - for (Row e : element.getValue()) { + Table table = openWriter.getOrCreateTable(destination, dataSchema); + Iterable sortedRows = + IcebergRowSorter.sortRows( + element.getValue(), table.sortOrder(), table.schema(), dataSchema); + for (Row e : sortedRows) { writer.write(windowedDestination, e); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java index 54ad120f1aca..4fd93ff60184 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java @@ -130,7 +130,10 @@ public void processElement( RecordWriter writer = new RecordWriter(table, destination.getFileFormat(), fileName, partitionData); try { - for (Row row : element.getValue()) { + Iterable sortedRows = + IcebergRowSorter.sortRows( + element.getValue(), table.sortOrder(), table.schema(), dataSchema); + for (Row row : sortedRows) { Record record = IcebergUtils.beamRowToIcebergRecord(table.schema(), row); writer.write(record); } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteUngroupedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteUngroupedRowsToFiles.java index 1db6ede30165..4d2dd1fe3d0d 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteUngroupedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteUngroupedRowsToFiles.java @@ -47,6 +47,7 @@ import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; +import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Catalog; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -238,11 +239,16 @@ public void processElement( WindowedValues.of(destination, window.maxTimestamp(), window, paneInfo); // Attempt to write record. If the writer is saturated and cannot accept - // the record, spill it over to WriteGroupedRowsToFiles - boolean writeSuccess; + // the record, or if the target table is sorted, spill it over to WriteGroupedRowsToFiles + boolean writeSuccess = false; try { - writeSuccess = - Preconditions.checkNotNull(recordWriterManager).write(windowedDestination, data); + Table table = + Preconditions.checkNotNull(recordWriterManager) + .getOrCreateTable(destination, data.getSchema()); + if (!table.sortOrder().isSorted()) { + writeSuccess = + Preconditions.checkNotNull(recordWriterManager).write(windowedDestination, data); + } } catch (Exception e) { try { Preconditions.checkNotNull(recordWriterManager).close(); diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java index 52d92911f4e4..c0a18dca0861 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java @@ -77,6 +77,7 @@ import org.apache.iceberg.DistributionMode; import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; @@ -774,4 +775,91 @@ public void process(@Element KV>> sums) { .getCommitted(); assertEquals(5L, numWaves); } + + @Test + public void testSortedWrite() { + TableIdentifier tableId = + TableIdentifier.of("default", "sorted_" + Long.toString(UUID.randomUUID().hashCode(), 16)); + + Map catalogProps = + ImmutableMap.builder() + .put("type", CatalogUtil.ICEBERG_CATALOG_TYPE_HADOOP) + .put("warehouse", warehouse.location) + .build(); + + IcebergCatalogConfig catalog = + IcebergCatalogConfig.builder() + .setCatalogName("name") + .setCatalogProperties(catalogProps) + .build(); + + Schema schema = Schema.builder().addInt64Field("id").addStringField("name").build(); + org.apache.iceberg.Schema icebergSchema = IcebergUtils.beamSchemaToIcebergSchema(schema); + + catalog + .catalog() + .buildTable(tableId, icebergSchema) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withSortOrder(SortOrder.builderFor(icebergSchema).asc("name").desc("id").build()) + .create(); + + List inputRows = + Arrays.asList( + Row.withSchema(schema).addValues(2L, "banana").build(), + Row.withSchema(schema).addValues(1L, "banana").build(), + Row.withSchema(schema).addValues(5L, "apple").build(), + Row.withSchema(schema).addValues(10L, "cherry").build()); + + testPipeline + .apply("Scrambled Input", Create.of(inputRows)) + .setRowSchema(schema) + .apply("Append Sorted To Table", writeTransform(catalog, tableId)); + + testPipeline.run().waitUntilFinish(); + + Table table = warehouse.loadTable(tableId); + List writtenRecords = ImmutableList.copyOf(IcebergGenerics.read(table).build()); + + assertEquals(4, writtenRecords.size()); + + try { + assertFilesAreInternallySorted(table); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void assertFilesAreInternallySorted(Table table) throws Exception { + for (org.apache.iceberg.FileScanTask task : table.newScan().planFiles()) { + String path = task.file().path().toString(); + try (org.apache.iceberg.io.CloseableIterable reader = + org.apache.iceberg.parquet.Parquet.read(table.io().newInputFile(path)) + .project(table.schema()) + .createReaderFunc(org.apache.iceberg.data.parquet.GenericParquetReaders::buildReader) + .build()) { + List records = + org.apache.commons.compress.utils.Lists.newArrayList(reader.iterator()); + assertTrue("File must have at least one record", records.size() > 0); + + for (int i = 1; i < records.size(); i++) { + Record prev = records.get(i - 1); + Record curr = records.get(i); + + String prevName = (String) prev.getField("name"); + String currName = (String) curr.getField("name"); + + int cmpName = prevName.compareTo(currName); + if (cmpName > 0) { + throw new AssertionError("File not sorted by name ASC: " + prevName + " > " + currName); + } else if (cmpName == 0) { + long prevId = (Long) prev.getField("id"); + long currId = (Long) curr.getField("id"); + if (prevId < currId) { + throw new AssertionError("File not sorted by id DESC: " + prevId + " < " + currId); + } + } + } + } + } + } } diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java new file mode 100644 index 000000000000..28cac8b65cee --- /dev/null +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.values.Row; +import org.apache.iceberg.NullOrder; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.types.Types; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class IcebergRowSorterTest { + + private static final Schema BEAM_SCHEMA = + Schema.builder() + .addInt32Field("id") + .addNullableField("name", Schema.FieldType.STRING) + .addNullableField("value", Schema.FieldType.DOUBLE) + .addNullableField("active", Schema.FieldType.BOOLEAN) + .build(); + + private static final org.apache.iceberg.Schema ICEBERG_SCHEMA = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "name", Types.StringType.get()), + Types.NestedField.optional(3, "value", Types.DoubleType.get()), + Types.NestedField.optional(4, "active", Types.BooleanType.get())); + + private static final Comparator BYTE_ARR_COMPARATOR = + org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes + .lexicographicalComparator(); + + @Test + public void testStringKeyEncodingOrder() throws Exception { + SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").build(); + + Row r1 = Row.withSchema(BEAM_SCHEMA).addValues(1, "apple", 1.5, true).build(); + Row r2 = Row.withSchema(BEAM_SCHEMA).addValues(2, "banana", 2.0, true).build(); + Row r3 = Row.withSchema(BEAM_SCHEMA).addValues(3, "apricot", 3.0, false).build(); + + byte[] k1 = IcebergRowSorter.encodeSortKey(r1, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k2 = IcebergRowSorter.encodeSortKey(r2, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k3 = IcebergRowSorter.encodeSortKey(r3, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + + assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k2) < 0); // apple < banana + assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k3) < 0); // apple < apricot + assertTrue(BYTE_ARR_COMPARATOR.compare(k3, k2) < 0); // apricot < banana + } + + @Test + public void testStringCollisionProofing() throws Exception { + // Tests that secondary columns don't bleed into primary columns. + // Row 1: Primary="abc", Secondary="def" + // Row 2: Primary="abcdef", Secondary=null + // In raw byte concatenation, both could equal "abcdef\0" if delimiters or escaping fail. + SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").asc("value").build(); + + Row r1 = Row.withSchema(BEAM_SCHEMA).addValues(1, "abc", 1.0, true).build(); + Row r2 = Row.withSchema(BEAM_SCHEMA).addValues(2, "abcdef", null, true).build(); + + byte[] k1 = IcebergRowSorter.encodeSortKey(r1, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k2 = IcebergRowSorter.encodeSortKey(r2, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + + // "abc" must sort lexicographically before "abcdef" + assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k2) < 0); + } + + @Test + public void testDescInversion() throws Exception { + SortOrder sortOrderAsc = SortOrder.builderFor(ICEBERG_SCHEMA).asc("id").build(); + SortOrder sortOrderDesc = SortOrder.builderFor(ICEBERG_SCHEMA).desc("id").build(); + + Row r1 = Row.withSchema(BEAM_SCHEMA).addValues(10, "test", 1.5, true).build(); + Row r2 = Row.withSchema(BEAM_SCHEMA).addValues(20, "test", 2.0, true).build(); + + byte[] k1Asc = IcebergRowSorter.encodeSortKey(r1, sortOrderAsc, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k2Asc = IcebergRowSorter.encodeSortKey(r2, sortOrderAsc, ICEBERG_SCHEMA, BEAM_SCHEMA); + + byte[] k1Desc = IcebergRowSorter.encodeSortKey(r1, sortOrderDesc, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k2Desc = IcebergRowSorter.encodeSortKey(r2, sortOrderDesc, ICEBERG_SCHEMA, BEAM_SCHEMA); + + // Ascending: 10 < 20 + assertTrue(BYTE_ARR_COMPARATOR.compare(k1Asc, k2Asc) < 0); + + // Descending: 10 > 20 (inverted bytes) + assertTrue(BYTE_ARR_COMPARATOR.compare(k1Desc, k2Desc) > 0); + } + + @Test + public void testNullOrderingMatrix() throws Exception { + Row rNonNull = Row.withSchema(BEAM_SCHEMA).addValues(1, "apple", 1.5, true).build(); + Row rNull = Row.withSchema(BEAM_SCHEMA).addValues(2, null, 2.0, true).build(); + + // 1. ASC, NULLS_FIRST + SortOrder ascFirst = + SortOrder.builderFor(ICEBERG_SCHEMA).asc("name", NullOrder.NULLS_FIRST).build(); + byte[] kNonNullAscFirst = + IcebergRowSorter.encodeSortKey(rNonNull, ascFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNullAscFirst = + IcebergRowSorter.encodeSortKey(rNull, ascFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); + assertTrue( + "ASC NULLS_FIRST failed: null should sort before non-null", + BYTE_ARR_COMPARATOR.compare(kNullAscFirst, kNonNullAscFirst) < 0); + + // 2. ASC, NULLS_LAST + SortOrder ascLast = + SortOrder.builderFor(ICEBERG_SCHEMA).asc("name", NullOrder.NULLS_LAST).build(); + byte[] kNonNullAscLast = + IcebergRowSorter.encodeSortKey(rNonNull, ascLast, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNullAscLast = + IcebergRowSorter.encodeSortKey(rNull, ascLast, ICEBERG_SCHEMA, BEAM_SCHEMA); + assertTrue( + "ASC NULLS_LAST failed: null should sort after non-null", + BYTE_ARR_COMPARATOR.compare(kNullAscLast, kNonNullAscLast) > 0); + + // 3. DESC, NULLS_FIRST + SortOrder descFirst = + SortOrder.builderFor(ICEBERG_SCHEMA).desc("name", NullOrder.NULLS_FIRST).build(); + byte[] kNonNullDescFirst = + IcebergRowSorter.encodeSortKey(rNonNull, descFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNullDescFirst = + IcebergRowSorter.encodeSortKey(rNull, descFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); + assertTrue( + "DESC NULLS_FIRST failed: null should sort before non-null", + BYTE_ARR_COMPARATOR.compare(kNullDescFirst, kNonNullDescFirst) > 0); + + // 4. DESC, NULLS_LAST + SortOrder descLast = + SortOrder.builderFor(ICEBERG_SCHEMA).desc("name", NullOrder.NULLS_LAST).build(); + byte[] kNonNullDescLast = + IcebergRowSorter.encodeSortKey(rNonNull, descLast, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNullDescLast = + IcebergRowSorter.encodeSortKey(rNull, descLast, ICEBERG_SCHEMA, BEAM_SCHEMA); + assertTrue( + "DESC NULLS_LAST failed: null should sort after non-null", + BYTE_ARR_COMPARATOR.compare(kNullDescLast, kNonNullDescLast) < 0); + } + + @Test + public void testEndToEndSorting() { + SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").desc("id").build(); + + List input = + Arrays.asList( + Row.withSchema(BEAM_SCHEMA).addValues(2, "banana", 2.0, true).build(), + Row.withSchema(BEAM_SCHEMA).addValues(1, "banana", 1.0, true).build(), + Row.withSchema(BEAM_SCHEMA).addValues(5, "apple", 1.5, true).build(), + Row.withSchema(BEAM_SCHEMA).addValues(10, "cherry", 3.0, false).build()); + + Iterable sorted = IcebergRowSorter.sortRows(input, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + List sortedList = + StreamSupport.stream(sorted.spliterator(), false).collect(Collectors.toList()); + + assertEquals(4, sortedList.size()); + + // Expected: apple (5) -> banana (2) -> banana (1) -> cherry (10) + assertEquals("apple", sortedList.get(0).getString("name")); + assertEquals(Integer.valueOf(5), sortedList.get(0).getInt32("id")); + + assertEquals("banana", sortedList.get(1).getString("name")); + assertEquals(Integer.valueOf(2), sortedList.get(1).getInt32("id")); + + assertEquals("banana", sortedList.get(2).getString("name")); + assertEquals(Integer.valueOf(1), sortedList.get(2).getInt32("id")); + + assertEquals("cherry", sortedList.get(3).getString("name")); + assertEquals(Integer.valueOf(10), sortedList.get(3).getInt32("id")); + } + + @Test + public void testScaleAndExternalDiskSpill() { + // Verifies sorting operates correctly with thousands of elements, + // proving that BufferedExternalSorter handles memory constraints correctly. + SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("id").build(); + + int count = 5000; + List input = new ArrayList<>(count); + Random rand = new Random(42); + + for (int i = 0; i < count; i++) { + // Intentionally insert random IDs to enforce complex sorting + int randomId = rand.nextInt(100_000); + input.add(Row.withSchema(BEAM_SCHEMA).addValues(randomId, "item" + i, 1.0, true).build()); + } + + Iterable sorted = IcebergRowSorter.sortRows(input, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + List sortedList = + StreamSupport.stream(sorted.spliterator(), false).collect(Collectors.toList()); + + assertEquals(count, sortedList.size()); + + // Validate that the returned dataset is in strictly non-decreasing order of 'id' + for (int i = 0; i < sortedList.size() - 1; i++) { + int idCurrent = sortedList.get(i).getInt32("id"); + int idNext = sortedList.get(i + 1).getInt32("id"); + assertTrue( + String.format("Sort violation detected at index %d: %d > %d", i, idCurrent, idNext), + idCurrent <= idNext); + } + } +} From 094c7210758eb84bc1eafc0b59f9519659518566 Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 17:49:31 +0200 Subject: [PATCH 02/21] Fix multiple-iteration anti-pattern in WriteGroupedRowsToFiles by extracting Schema from Coder --- .../io/iceberg/WriteGroupedRowsToFiles.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java index dde519128f63..349313e4210f 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java @@ -17,8 +17,10 @@ */ package org.apache.beam.sdk.io.iceberg; -import java.util.Iterator; import java.util.List; +import org.apache.beam.sdk.coders.IterableCoder; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; @@ -59,10 +61,17 @@ class WriteGroupedRowsToFiles @Override public PCollection expand( PCollection, Iterable>> input) { + Schema dataSchema = + ((RowCoder) + ((IterableCoder) + ((KvCoder, Iterable>) input.getCoder()) + .getValueCoder()) + .getElemCoder()) + .getSchema(); return input.apply( ParDo.of( new WriteGroupedRowsToFilesDoFn( - catalogConfig, dynamicDestinations, maxBytesPerFile, filePrefix))); + catalogConfig, dynamicDestinations, maxBytesPerFile, filePrefix, dataSchema))); } private static class WriteGroupedRowsToFilesDoFn @@ -73,16 +82,19 @@ private static class WriteGroupedRowsToFilesDoFn private transient @MonotonicNonNull Catalog catalog; private final String filePrefix; private final long maxFileSize; + private final Schema dataSchema; WriteGroupedRowsToFilesDoFn( IcebergCatalogConfig catalogConfig, DynamicDestinations dynamicDestinations, long maxFileSize, - String filePrefix) { + String filePrefix, + Schema dataSchema) { this.catalogConfig = catalogConfig; this.dynamicDestinations = dynamicDestinations; this.filePrefix = filePrefix; this.maxFileSize = maxFileSize; + this.dataSchema = dataSchema; } private org.apache.iceberg.catalog.Catalog getCatalog() { @@ -104,12 +116,6 @@ public void processElement( IcebergDestination destination = dynamicDestinations.instantiateDestination(tableIdentifier); WindowedValue windowedDestination = WindowedValues.of(destination, window.maxTimestamp(), window, paneInfo); - Iterator rowIt = element.getValue().iterator(); - if (!rowIt.hasNext()) { - return; - } - Row firstRow = rowIt.next(); - Schema dataSchema = firstRow.getSchema(); RecordWriterManager writer; try (RecordWriterManager openWriter = From f9dd27d0df3be0541234892ebcb54aa987fae4b3 Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 17:58:32 +0200 Subject: [PATCH 03/21] Refactor row sorting calls to be explicitly conditional on table.sortOrder().isSorted() for readability --- .../beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java | 10 ++++++---- .../sdk/io/iceberg/WritePartitionedRowsToFiles.java | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java index 349313e4210f..b9092afe3774 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java @@ -122,10 +122,12 @@ public void processElement( new RecordWriterManager(getCatalog(), filePrefix, maxFileSize, Integer.MAX_VALUE)) { writer = openWriter; Table table = openWriter.getOrCreateTable(destination, dataSchema); - Iterable sortedRows = - IcebergRowSorter.sortRows( - element.getValue(), table.sortOrder(), table.schema(), dataSchema); - for (Row e : sortedRows) { + Iterable rowsToWrite = element.getValue(); + if (table.sortOrder().isSorted()) { + rowsToWrite = + IcebergRowSorter.sortRows(rowsToWrite, table.sortOrder(), table.schema(), dataSchema); + } + for (Row e : rowsToWrite) { writer.write(windowedDestination, e); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java index 4fd93ff60184..f661152fc7ca 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java @@ -130,10 +130,12 @@ public void processElement( RecordWriter writer = new RecordWriter(table, destination.getFileFormat(), fileName, partitionData); try { - Iterable sortedRows = - IcebergRowSorter.sortRows( - element.getValue(), table.sortOrder(), table.schema(), dataSchema); - for (Row row : sortedRows) { + Iterable rowsToWrite = element.getValue(); + if (table.sortOrder().isSorted()) { + rowsToWrite = + IcebergRowSorter.sortRows(rowsToWrite, table.sortOrder(), table.schema(), dataSchema); + } + for (Row row : rowsToWrite) { Record record = IcebergUtils.beamRowToIcebergRecord(table.schema(), row); writer.write(record); } From ca4ff74bfa31ff74b31300d3c48b8f78b28c16b2 Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 18:04:04 +0200 Subject: [PATCH 04/21] Simplify sorting logic to keep call DRY and use straightforward 'rows' variable name --- .../beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java | 10 ++++------ .../sdk/io/iceberg/WritePartitionedRowsToFiles.java | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java index b9092afe3774..e4b5dbdb3f3f 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java @@ -122,12 +122,10 @@ public void processElement( new RecordWriterManager(getCatalog(), filePrefix, maxFileSize, Integer.MAX_VALUE)) { writer = openWriter; Table table = openWriter.getOrCreateTable(destination, dataSchema); - Iterable rowsToWrite = element.getValue(); - if (table.sortOrder().isSorted()) { - rowsToWrite = - IcebergRowSorter.sortRows(rowsToWrite, table.sortOrder(), table.schema(), dataSchema); - } - for (Row e : rowsToWrite) { + Iterable rows = + IcebergRowSorter.sortRows( + element.getValue(), table.sortOrder(), table.schema(), dataSchema); + for (Row e : rows) { writer.write(windowedDestination, e); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java index f661152fc7ca..fd93163f967c 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java @@ -130,12 +130,10 @@ public void processElement( RecordWriter writer = new RecordWriter(table, destination.getFileFormat(), fileName, partitionData); try { - Iterable rowsToWrite = element.getValue(); - if (table.sortOrder().isSorted()) { - rowsToWrite = - IcebergRowSorter.sortRows(rowsToWrite, table.sortOrder(), table.schema(), dataSchema); - } - for (Row row : rowsToWrite) { + Iterable rows = + IcebergRowSorter.sortRows( + element.getValue(), table.sortOrder(), table.schema(), dataSchema); + for (Row row : rows) { Record record = IcebergUtils.beamRowToIcebergRecord(table.schema(), row); writer.write(record); } From df8dc69beaed338b2729b300f463515e8bdae74c Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 18:05:53 +0200 Subject: [PATCH 05/21] Rename variable rows to sortedOrUnsortedRows to explicitly communicate its conditional state --- .../apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java | 4 ++-- .../beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java index e4b5dbdb3f3f..95384dea1887 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WriteGroupedRowsToFiles.java @@ -122,10 +122,10 @@ public void processElement( new RecordWriterManager(getCatalog(), filePrefix, maxFileSize, Integer.MAX_VALUE)) { writer = openWriter; Table table = openWriter.getOrCreateTable(destination, dataSchema); - Iterable rows = + Iterable sortedOrUnsortedRows = IcebergRowSorter.sortRows( element.getValue(), table.sortOrder(), table.schema(), dataSchema); - for (Row e : rows) { + for (Row e : sortedOrUnsortedRows) { writer.write(windowedDestination, e); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java index fd93163f967c..92c20433c462 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java @@ -130,10 +130,10 @@ public void processElement( RecordWriter writer = new RecordWriter(table, destination.getFileFormat(), fileName, partitionData); try { - Iterable rows = + Iterable sortedOrUnsortedRows = IcebergRowSorter.sortRows( element.getValue(), table.sortOrder(), table.schema(), dataSchema); - for (Row row : rows) { + for (Row row : sortedOrUnsortedRows) { Record record = IcebergUtils.beamRowToIcebergRecord(table.schema(), row); writer.write(record); } From 9120fda70b0891fb7dae8299935f5f7a5f1bf03e Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 18:19:44 +0200 Subject: [PATCH 06/21] Fix sorted write PR comments: resolve null-ordering bugs and optimize key encoding performance --- .../beam/sdk/io/iceberg/IcebergRowSorter.java | 174 +++++++++++------- .../sdk/io/iceberg/IcebergRowSorterTest.java | 55 +++--- 2 files changed, 139 insertions(+), 90 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java index 96e35a9cdcce..10d97062f1be 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java @@ -26,6 +26,7 @@ import java.time.Instant; import java.util.Date; import java.util.Iterator; +import java.util.List; import org.apache.beam.sdk.coders.RowCoder; import org.apache.beam.sdk.extensions.sorter.BufferedExternalSorter; import org.apache.beam.sdk.values.KV; @@ -35,7 +36,6 @@ import org.apache.iceberg.SortDirection; import org.apache.iceberg.SortField; import org.apache.iceberg.SortOrder; -import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.ReadableInstant; /** @@ -58,9 +58,15 @@ public static Iterable sortRows( BufferedExternalSorter sorter = BufferedExternalSorter.create(sorterOptions); RowCoder rowCoder = RowCoder.of(beamSchema); + List fields = sortOrder.fields(); + String[] columnNames = new String[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + columnNames[i] = icebergSchema.findColumnName(fields.get(i).sourceId()); + } + try { for (Row row : rows) { - byte[] keyBytes = encodeSortKey(row, sortOrder, icebergSchema, beamSchema); + byte[] keyBytes = encodeSortKey(row, sortOrder, columnNames, icebergSchema, beamSchema); ByteArrayOutputStream baos = new ByteArrayOutputStream(); rowCoder.encode(row, baos); byte[] valBytes = baos.toByteArray(); @@ -97,6 +103,7 @@ public Row next() { } } + @Deprecated @SuppressWarnings("nullness") public static byte[] encodeSortKey( Row row, @@ -104,16 +111,37 @@ public static byte[] encodeSortKey( Schema icebergSchema, org.apache.beam.sdk.schemas.Schema beamSchema) throws IOException { + List fields = sortOrder.fields(); + String[] columnNames = new String[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + columnNames[i] = icebergSchema.findColumnName(fields.get(i).sourceId()); + } + return encodeSortKey(row, sortOrder, columnNames, icebergSchema, beamSchema); + } + + @SuppressWarnings("nullness") + public static byte[] encodeSortKey( + Row row, + SortOrder sortOrder, + String[] columnNames, + Schema icebergSchema, + org.apache.beam.sdk.schemas.Schema beamSchema) + throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); + List fields = sortOrder.fields(); + org.apache.iceberg.data.Record icebergRecord = null; - for (SortField field : sortOrder.fields()) { - String colName = icebergSchema.findColumnName(field.sourceId()); + for (int i = 0; i < fields.size(); i++) { + SortField field = fields.get(i); + String colName = columnNames[i]; Object val = row.getValue(colName); if (!field.transform().isIdentity()) { - Object icebergVal = - IcebergUtils.beamRowToIcebergRecord(icebergSchema, row).getField(colName); + if (icebergRecord == null) { + icebergRecord = IcebergUtils.beamRowToIcebergRecord(icebergSchema, row); + } + Object icebergVal = icebergRecord.getField(colName); if (icebergVal != null) { val = field.transform().apply(icebergVal); } else { @@ -128,107 +156,127 @@ public static byte[] encodeSortKey( // Determine correct header prefix to fulfill the NullOrder contracts byte prefixByte; if (isNull) { - if (isDesc) { - // Descending: High byte keys sort first. - // If Nulls First -> Null gets highest byte (0xFF) - // If Nulls Last -> Null gets lowest byte (0x00) - prefixByte = nullsFirst ? (byte) 0xFF : (byte) 0x00; - } else { - // Ascending: Low byte keys sort first. - // If Nulls First -> Null gets lowest byte (0x00) - // If Nulls Last -> Null gets highest byte (0xFF) - prefixByte = nullsFirst ? (byte) 0x00 : (byte) 0xFF; - } + prefixByte = nullsFirst ? (byte) 0x00 : (byte) 0xFF; } else { - if (isDesc) { - // If non-null and Descending, use a neutral value that sits opposite to the null byte - prefixByte = nullsFirst ? (byte) 0xFE : (byte) 0x01; - } else { - prefixByte = nullsFirst ? (byte) 0x01 : (byte) 0x00; - } + prefixByte = nullsFirst ? (byte) 0x01 : (byte) 0x00; } baos.write(prefixByte); if (!isNull) { - byte[] valBytes = encodeValue(val); - // Bitwise invert non-null bytes to sort descending lexicographically - if (isDesc) { - for (int i = 0; i < valBytes.length; i++) { - valBytes[i] = (byte) ~valBytes[i]; - } - } - baos.write(valBytes); + writeValue(val, baos, isDesc); } } return baos.toByteArray(); } - @SuppressWarnings("JavaUtilDate") - private static byte[] encodeValue(@Nullable Object val) throws IOException { - if (val == null) { - return new byte[0]; + private static void writeInt(int v, ByteArrayOutputStream baos, boolean invert) { + byte b3 = (byte) (v >>> 24); + byte b2 = (byte) (v >>> 16); + byte b1 = (byte) (v >>> 8); + byte b0 = (byte) v; + if (invert) { + baos.write(~b3); + baos.write(~b2); + baos.write(~b1); + baos.write(~b0); + } else { + baos.write(b3); + baos.write(b2); + baos.write(b1); + baos.write(b0); + } + } + + private static void writeLong(long v, ByteArrayOutputStream baos, boolean invert) { + byte b7 = (byte) (v >>> 56); + byte b6 = (byte) (v >>> 48); + byte b5 = (byte) (v >>> 40); + byte b4 = (byte) (v >>> 32); + byte b3 = (byte) (v >>> 24); + byte b2 = (byte) (v >>> 16); + byte b1 = (byte) (v >>> 8); + byte b0 = (byte) v; + if (invert) { + baos.write(~b7); + baos.write(~b6); + baos.write(~b5); + baos.write(~b4); + baos.write(~b3); + baos.write(~b2); + baos.write(~b1); + baos.write(~b0); + } else { + baos.write(b7); + baos.write(b6); + baos.write(b5); + baos.write(b4); + baos.write(b3); + baos.write(b2); + baos.write(b1); + baos.write(b0); } + } + + @SuppressWarnings("JavaUtilDate") + private static void writeValue(Object val, ByteArrayOutputStream baos, boolean invert) + throws IOException { if (val instanceof String) { - return encodeString((String) val); + writeString((String) val, baos, invert); } else if (val instanceof Integer) { int v = (Integer) val; - return ByteBuffer.allocate(4).putInt(v ^ Integer.MIN_VALUE).array(); + writeInt(v ^ Integer.MIN_VALUE, baos, invert); } else if (val instanceof Long) { long v = (Long) val; - return ByteBuffer.allocate(8).putLong(v ^ Long.MIN_VALUE).array(); + writeLong(v ^ Long.MIN_VALUE, baos, invert); } else if (val instanceof Float) { int bits = Float.floatToIntBits((Float) val); bits = (bits >= 0) ? (bits ^ Integer.MIN_VALUE) : ~bits; - return ByteBuffer.allocate(4).putInt(bits).array(); + writeInt(bits, baos, invert); } else if (val instanceof Double) { long bits = Double.doubleToLongBits((Double) val); bits = (bits >= 0) ? (bits ^ Long.MIN_VALUE) : ~bits; - return ByteBuffer.allocate(8).putLong(bits).array(); + writeLong(bits, baos, invert); } else if (val instanceof Boolean) { - return new byte[] {((Boolean) val) ? (byte) 0x01 : (byte) 0x00}; + byte b = ((Boolean) val) ? (byte) 0x01 : (byte) 0x00; + baos.write(invert ? ~b : b); } else if (val instanceof byte[]) { - return encodeByteArray((byte[]) val); + writeByteArray((byte[]) val, baos, invert); } else if (val instanceof ByteBuffer) { - return encodeByteArray(((ByteBuffer) val).array()); + writeByteArray(((ByteBuffer) val).array(), baos, invert); } else if (val instanceof ReadableInstant) { long enc = ((ReadableInstant) val).getMillis() ^ Long.MIN_VALUE; - return ByteBuffer.allocate(8).putLong(enc).array(); + writeLong(enc, baos, invert); } else if (val instanceof Instant) { long enc = ((Instant) val).toEpochMilli() ^ Long.MIN_VALUE; - return ByteBuffer.allocate(8).putLong(enc).array(); + writeLong(enc, baos, invert); } else if (val instanceof Date) { long enc = ((Date) val).getTime() ^ Long.MIN_VALUE; - return ByteBuffer.allocate(8).putLong(enc).array(); + writeLong(enc, baos, invert); + } else { + writeString(val.toString(), baos, invert); } - - return encodeString(val.toString()); } - private static byte[] encodeString(String s) throws IOException { + private static void writeString(String s, ByteArrayOutputStream baos, boolean invert) + throws IOException { byte[] bytes = s.getBytes(StandardCharsets.UTF_8); - return encodeByteArray(bytes); + writeByteArray(bytes, baos, invert); } - /** - * Escape protocol to cleanly prevent collisions. Maps 0x00 -> [0x01, 0x01] Maps 0x01 -> [0x01, - * 0x02] Safely terminates sequence with 0x00. - */ - private static byte[] encodeByteArray(byte[] bytes) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(bytes.length + 2); + private static void writeByteArray(byte[] bytes, ByteArrayOutputStream baos, boolean invert) { for (byte b : bytes) { if (b == 0x00) { - baos.write(0x01); - baos.write(0x01); + baos.write(invert ? ~(byte) 0x01 : (byte) 0x01); + baos.write(invert ? ~(byte) 0x01 : (byte) 0x01); } else if (b == 0x01) { - baos.write(0x01); - baos.write(0x02); + baos.write(invert ? ~(byte) 0x01 : (byte) 0x01); + baos.write(invert ? ~(byte) 0x02 : (byte) 0x02); } else { - baos.write(b); + baos.write(invert ? ~b : b); } } - baos.write(0x00); // Safe boundary delimiter - return baos.toByteArray(); + baos.write(invert ? ~(byte) 0x00 : (byte) 0x00); } } diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java index 28cac8b65cee..ebb50981bd9d 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java @@ -58,6 +58,15 @@ public class IcebergRowSorterTest { org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedBytes .lexicographicalComparator(); + private static byte[] encodeSortKeyHelper(Row row, SortOrder sortOrder) throws Exception { + java.util.List fields = sortOrder.fields(); + String[] columnNames = new String[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + columnNames[i] = ICEBERG_SCHEMA.findColumnName(fields.get(i).sourceId()); + } + return IcebergRowSorter.encodeSortKey(row, sortOrder, columnNames, ICEBERG_SCHEMA, BEAM_SCHEMA); + } + @Test public void testStringKeyEncodingOrder() throws Exception { SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").build(); @@ -66,9 +75,9 @@ public void testStringKeyEncodingOrder() throws Exception { Row r2 = Row.withSchema(BEAM_SCHEMA).addValues(2, "banana", 2.0, true).build(); Row r3 = Row.withSchema(BEAM_SCHEMA).addValues(3, "apricot", 3.0, false).build(); - byte[] k1 = IcebergRowSorter.encodeSortKey(r1, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] k2 = IcebergRowSorter.encodeSortKey(r2, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] k3 = IcebergRowSorter.encodeSortKey(r3, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k1 = encodeSortKeyHelper(r1, sortOrder); + byte[] k2 = encodeSortKeyHelper(r2, sortOrder); + byte[] k3 = encodeSortKeyHelper(r3, sortOrder); assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k2) < 0); // apple < banana assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k3) < 0); // apple < apricot @@ -86,8 +95,8 @@ public void testStringCollisionProofing() throws Exception { Row r1 = Row.withSchema(BEAM_SCHEMA).addValues(1, "abc", 1.0, true).build(); Row r2 = Row.withSchema(BEAM_SCHEMA).addValues(2, "abcdef", null, true).build(); - byte[] k1 = IcebergRowSorter.encodeSortKey(r1, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] k2 = IcebergRowSorter.encodeSortKey(r2, sortOrder, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k1 = encodeSortKeyHelper(r1, sortOrder); + byte[] k2 = encodeSortKeyHelper(r2, sortOrder); // "abc" must sort lexicographically before "abcdef" assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k2) < 0); @@ -101,11 +110,11 @@ public void testDescInversion() throws Exception { Row r1 = Row.withSchema(BEAM_SCHEMA).addValues(10, "test", 1.5, true).build(); Row r2 = Row.withSchema(BEAM_SCHEMA).addValues(20, "test", 2.0, true).build(); - byte[] k1Asc = IcebergRowSorter.encodeSortKey(r1, sortOrderAsc, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] k2Asc = IcebergRowSorter.encodeSortKey(r2, sortOrderAsc, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k1Asc = encodeSortKeyHelper(r1, sortOrderAsc); + byte[] k2Asc = encodeSortKeyHelper(r2, sortOrderAsc); - byte[] k1Desc = IcebergRowSorter.encodeSortKey(r1, sortOrderDesc, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] k2Desc = IcebergRowSorter.encodeSortKey(r2, sortOrderDesc, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] k1Desc = encodeSortKeyHelper(r1, sortOrderDesc); + byte[] k2Desc = encodeSortKeyHelper(r2, sortOrderDesc); // Ascending: 10 < 20 assertTrue(BYTE_ARR_COMPARATOR.compare(k1Asc, k2Asc) < 0); @@ -122,10 +131,8 @@ public void testNullOrderingMatrix() throws Exception { // 1. ASC, NULLS_FIRST SortOrder ascFirst = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name", NullOrder.NULLS_FIRST).build(); - byte[] kNonNullAscFirst = - IcebergRowSorter.encodeSortKey(rNonNull, ascFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] kNullAscFirst = - IcebergRowSorter.encodeSortKey(rNull, ascFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNonNullAscFirst = encodeSortKeyHelper(rNonNull, ascFirst); + byte[] kNullAscFirst = encodeSortKeyHelper(rNull, ascFirst); assertTrue( "ASC NULLS_FIRST failed: null should sort before non-null", BYTE_ARR_COMPARATOR.compare(kNullAscFirst, kNonNullAscFirst) < 0); @@ -133,10 +140,8 @@ public void testNullOrderingMatrix() throws Exception { // 2. ASC, NULLS_LAST SortOrder ascLast = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name", NullOrder.NULLS_LAST).build(); - byte[] kNonNullAscLast = - IcebergRowSorter.encodeSortKey(rNonNull, ascLast, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] kNullAscLast = - IcebergRowSorter.encodeSortKey(rNull, ascLast, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNonNullAscLast = encodeSortKeyHelper(rNonNull, ascLast); + byte[] kNullAscLast = encodeSortKeyHelper(rNull, ascLast); assertTrue( "ASC NULLS_LAST failed: null should sort after non-null", BYTE_ARR_COMPARATOR.compare(kNullAscLast, kNonNullAscLast) > 0); @@ -144,24 +149,20 @@ public void testNullOrderingMatrix() throws Exception { // 3. DESC, NULLS_FIRST SortOrder descFirst = SortOrder.builderFor(ICEBERG_SCHEMA).desc("name", NullOrder.NULLS_FIRST).build(); - byte[] kNonNullDescFirst = - IcebergRowSorter.encodeSortKey(rNonNull, descFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] kNullDescFirst = - IcebergRowSorter.encodeSortKey(rNull, descFirst, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNonNullDescFirst = encodeSortKeyHelper(rNonNull, descFirst); + byte[] kNullDescFirst = encodeSortKeyHelper(rNull, descFirst); assertTrue( "DESC NULLS_FIRST failed: null should sort before non-null", - BYTE_ARR_COMPARATOR.compare(kNullDescFirst, kNonNullDescFirst) > 0); + BYTE_ARR_COMPARATOR.compare(kNullDescFirst, kNonNullDescFirst) < 0); // 4. DESC, NULLS_LAST SortOrder descLast = SortOrder.builderFor(ICEBERG_SCHEMA).desc("name", NullOrder.NULLS_LAST).build(); - byte[] kNonNullDescLast = - IcebergRowSorter.encodeSortKey(rNonNull, descLast, ICEBERG_SCHEMA, BEAM_SCHEMA); - byte[] kNullDescLast = - IcebergRowSorter.encodeSortKey(rNull, descLast, ICEBERG_SCHEMA, BEAM_SCHEMA); + byte[] kNonNullDescLast = encodeSortKeyHelper(rNonNull, descLast); + byte[] kNullDescLast = encodeSortKeyHelper(rNull, descLast); assertTrue( "DESC NULLS_LAST failed: null should sort after non-null", - BYTE_ARR_COMPARATOR.compare(kNullDescLast, kNonNullDescLast) < 0); + BYTE_ARR_COMPARATOR.compare(kNullDescLast, kNonNullDescLast) > 0); } @Test From 75dcc4f346d85343ec83537721d8bb88013519fa Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 18:37:21 +0200 Subject: [PATCH 07/21] Fix Checkstyle check MissingDeprecated warning in IcebergRowSorter.java --- .../java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java index 10d97062f1be..708fda531107 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java @@ -103,6 +103,10 @@ public Row next() { } } + /** + * @deprecated Use {@link #encodeSortKey(Row, SortOrder, String[], Schema, + * org.apache.beam.sdk.schemas.Schema)} instead. + */ @Deprecated @SuppressWarnings("nullness") public static byte[] encodeSortKey( From e4c97bf7de5a42cd9a6f76cec7724058bbeccb96 Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Thu, 7 May 2026 18:40:49 +0200 Subject: [PATCH 08/21] Remove unused deprecated encodeSortKey method in IcebergRowSorter.java --- .../beam/sdk/io/iceberg/IcebergRowSorter.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java index 708fda531107..c7173274109b 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java @@ -103,26 +103,6 @@ public Row next() { } } - /** - * @deprecated Use {@link #encodeSortKey(Row, SortOrder, String[], Schema, - * org.apache.beam.sdk.schemas.Schema)} instead. - */ - @Deprecated - @SuppressWarnings("nullness") - public static byte[] encodeSortKey( - Row row, - SortOrder sortOrder, - Schema icebergSchema, - org.apache.beam.sdk.schemas.Schema beamSchema) - throws IOException { - List fields = sortOrder.fields(); - String[] columnNames = new String[fields.size()]; - for (int i = 0; i < fields.size(); i++) { - columnNames[i] = icebergSchema.findColumnName(fields.get(i).sourceId()); - } - return encodeSortKey(row, sortOrder, columnNames, icebergSchema, beamSchema); - } - @SuppressWarnings("nullness") public static byte[] encodeSortKey( Row row, From ecdaf7f049c93e39f2c1e7de9462513dd8bb3dab Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Fri, 8 May 2026 13:04:55 +0200 Subject: [PATCH 09/21] Finalize Hybrid Partitioning and Sorting Architecture in IcebergIO: expose NONE, HASH, and RANGE modes, optimize stream reuse, and add comprehensive sorting tests --- .../AssignDestinationsAndPartitions.java | 44 +++++++++++++++++-- .../apache/beam/sdk/io/iceberg/IcebergIO.java | 37 ++++++++++++++-- .../beam/sdk/io/iceberg/IcebergRowSorter.java | 31 +++++++------ .../beam/sdk/io/iceberg/IcebergUtils.java | 39 ++++++++++++++++ .../sdk/io/iceberg/IcebergRowSorterTest.java | 36 ++++++++------- 5 files changed, 152 insertions(+), 35 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java index 475786d3a4f6..df2d2c5433ec 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java @@ -26,12 +26,14 @@ import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.PaneInfo; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.ValueInSingleWindow; +import org.apache.iceberg.DistributionMode; import org.apache.iceberg.PartitionKey; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -51,38 +53,65 @@ class AssignDestinationsAndPartitions private final DynamicDestinations dynamicDestinations; private final IcebergCatalogConfig catalogConfig; + private final DistributionMode distributionMode; + private final @Nullable SerializableFunction distributionFunction; + static final String DESTINATION = "destination"; static final String PARTITION = "partition"; + static final String SHARD = "shard"; static final org.apache.beam.sdk.schemas.Schema OUTPUT_SCHEMA = org.apache.beam.sdk.schemas.Schema.builder() .addStringField(DESTINATION) .addStringField(PARTITION) + .addNullableField(SHARD, org.apache.beam.sdk.schemas.Schema.FieldType.INT32) .build(); public AssignDestinationsAndPartitions( DynamicDestinations dynamicDestinations, IcebergCatalogConfig catalogConfig) { + this(dynamicDestinations, catalogConfig, DistributionMode.HASH, null); + } + + public AssignDestinationsAndPartitions( + DynamicDestinations dynamicDestinations, + IcebergCatalogConfig catalogConfig, + DistributionMode distributionMode, + @Nullable SerializableFunction distributionFunction) { this.dynamicDestinations = dynamicDestinations; this.catalogConfig = catalogConfig; + this.distributionMode = distributionMode; + this.distributionFunction = distributionFunction; } @Override public PCollection> expand(PCollection input) { return input - .apply(ParDo.of(new AssignDoFn(dynamicDestinations, catalogConfig))) + .apply( + ParDo.of( + new AssignDoFn( + dynamicDestinations, catalogConfig, distributionMode, distributionFunction))) .setCoder( KvCoder.of( RowCoder.of(OUTPUT_SCHEMA), RowCoder.of(dynamicDestinations.getDataSchema()))); } + @SuppressWarnings("nullness") static class AssignDoFn extends DoFn> { private transient @MonotonicNonNull Map partitionKeys; private transient @MonotonicNonNull Map wrappers; private final DynamicDestinations dynamicDestinations; private final IcebergCatalogConfig catalogConfig; + private final DistributionMode distributionMode; + private final @Nullable SerializableFunction distributionFunction; - AssignDoFn(DynamicDestinations dynamicDestinations, IcebergCatalogConfig catalogConfig) { + AssignDoFn( + DynamicDestinations dynamicDestinations, + IcebergCatalogConfig catalogConfig, + DistributionMode distributionMode, + @Nullable SerializableFunction distributionFunction) { this.dynamicDestinations = dynamicDestinations; this.catalogConfig = catalogConfig; + this.distributionMode = distributionMode; + this.distributionFunction = distributionFunction; } @Setup @@ -132,8 +161,17 @@ public void processElement( partitionKey.partition(wrapper.wrap(data)); String partitionPath = partitionKey.toPath(); + Integer shardId = null; + if (distributionMode == DistributionMode.RANGE && distributionFunction != null) { + shardId = distributionFunction.apply(data); + } + Row destAndPartition = - Row.withSchema(OUTPUT_SCHEMA).addValues(tableIdentifier, partitionPath).build(); + Row.withSchema(OUTPUT_SCHEMA) + .withFieldValue(DESTINATION, tableIdentifier) + .withFieldValue(PARTITION, partitionPath) + .withFieldValue(SHARD, shardId) + .build(); out.output(KV.of(destAndPartition, data)); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index a5a3beef8f51..7962beb39f14 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -26,6 +26,7 @@ import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; @@ -404,6 +405,8 @@ public abstract static class WriteRows extends PTransform, Iceb abstract DistributionMode getDistributionMode(); + abstract @Nullable SerializableFunction getDistributionFunction(); + abstract boolean getAutoSharding(); abstract Builder toBuilder(); @@ -422,6 +425,8 @@ abstract static class Builder { abstract Builder setDistributionMode(DistributionMode mode); + abstract Builder setDistributionFunction(SerializableFunction shardFn); + abstract Builder setAutoSharding(boolean autoSharding); abstract WriteRows build(); @@ -462,14 +467,17 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) { *
    *
  1. {@link DistributionMode.NONE}: don't shuffle rows (default) *
  2. {@link DistributionMode.HASH}: shuffle rows by partition key before writing data + *
  3. {@link DistributionMode.RANGE}: shuffle rows based on range-partitioning function *
- * - * {@link DistributionMode.RANGE} is not supported yet */ public WriteRows withDistributionMode(DistributionMode mode) { return toBuilder().setDistributionMode(mode).build(); } + public WriteRows withDistributionFunction(SerializableFunction shardFn) { + return toBuilder().setDistributionFunction(shardFn).build(); + } + public WriteRows withAutosharding() { return toBuilder().setAutoSharding(true).build(); } @@ -514,7 +522,30 @@ public IcebergWriteResult expand(PCollection input) { return input .apply( "AssignDestinationAndPartition", - new AssignDestinationsAndPartitions(destinations, getCatalogConfig())) + new AssignDestinationsAndPartitions( + destinations, + getCatalogConfig(), + getDistributionMode(), + getDistributionFunction())) + .apply( + "Write Rows to Partitions", + new WriteToPartitions( + getCatalogConfig(), + destinations, + getTriggeringFrequency(), + getAutoSharding())); + case RANGE: + Preconditions.checkArgument( + getDistributionFunction() != null, + "Must provide a distribution function when using RANGE distribution mode."); + return input + .apply( + "AssignDestinationAndPartitionWithRange", + new AssignDestinationsAndPartitions( + destinations, + getCatalogConfig(), + getDistributionMode(), + getDistributionFunction())) .apply( "Write Rows to Partitions", new WriteToPartitions( diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java index c7173274109b..6efc1bbe2eec 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java @@ -64,12 +64,19 @@ public static Iterable sortRows( columnNames[i] = icebergSchema.findColumnName(fields.get(i).sourceId()); } + // Create reusable ByteArrayOutputStreams for key and value encoding + ByteArrayOutputStream keyBaos = new ByteArrayOutputStream(); + ByteArrayOutputStream valBaos = new ByteArrayOutputStream(); + try { for (Row row : rows) { - byte[] keyBytes = encodeSortKey(row, sortOrder, columnNames, icebergSchema, beamSchema); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - rowCoder.encode(row, baos); - byte[] valBytes = baos.toByteArray(); + keyBaos.reset(); + valBaos.reset(); + encodeSortKey(row, sortOrder, columnNames, keyBaos, icebergSchema, beamSchema); + byte[] keyBytes = keyBaos.toByteArray(); + + rowCoder.encode(row, valBaos); + byte[] valBytes = valBaos.toByteArray(); sorter.add(KV.of(keyBytes, valBytes)); } @@ -104,17 +111,16 @@ public Row next() { } @SuppressWarnings("nullness") - public static byte[] encodeSortKey( + public static void encodeSortKey( Row row, SortOrder sortOrder, String[] columnNames, + ByteArrayOutputStream baos, Schema icebergSchema, org.apache.beam.sdk.schemas.Schema beamSchema) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); List fields = sortOrder.fields(); - org.apache.iceberg.data.Record icebergRecord = null; for (int i = 0; i < fields.size(); i++) { SortField field = fields.get(i); @@ -122,10 +128,8 @@ public static byte[] encodeSortKey( Object val = row.getValue(colName); if (!field.transform().isIdentity()) { - if (icebergRecord == null) { - icebergRecord = IcebergUtils.beamRowToIcebergRecord(icebergSchema, row); - } - Object icebergVal = icebergRecord.getField(colName); + Object icebergVal = + IcebergUtils.beamValueToIcebergValue(icebergSchema.findType(field.sourceId()), val); if (icebergVal != null) { val = field.transform().apply(icebergVal); } else { @@ -151,8 +155,6 @@ public static byte[] encodeSortKey( writeValue(val, baos, isDesc); } } - - return baos.toByteArray(); } private static void writeInt(int v, ByteArrayOutputStream baos, boolean invert) { @@ -239,7 +241,8 @@ private static void writeValue(Object val, ByteArrayOutputStream baos, boolean i long enc = ((Date) val).getTime() ^ Long.MIN_VALUE; writeLong(enc, baos, invert); } else { - writeString(val.toString(), baos, invert); + throw new UnsupportedOperationException( + "Unsupported type for sorting: " + val.getClass().getName()); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java index d0d24532ff39..0e8def1656d5 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java @@ -296,6 +296,45 @@ public static org.apache.iceberg.Schema beamSchemaToIcebergSchema(final Schema s return new org.apache.iceberg.Schema(fields.toArray(new Types.NestedField[fields.size()])); } + /** + * Converts a Beam field value to its Iceberg-compatible equivalent based on the Iceberg {@link + * Type}. + */ + public static @Nullable Object beamValueToIcebergValue(Type type, @Nullable Object value) { + if (value == null) { + return null; + } + switch (type.typeId()) { + case BOOLEAN: + case INTEGER: + case LONG: + case FLOAT: + case DOUBLE: + case DATE: + case TIME: + case DECIMAL: + case STRING: + return value; + case TIMESTAMP: + Types.TimestampType ts = (Types.TimestampType) type.asPrimitiveType(); + return getIcebergTimestampValue(value, ts.shouldAdjustToUTC()); + case UUID: + if (value instanceof byte[]) { + return UUID.nameUUIDFromBytes((byte[]) value); + } + return value; + case BINARY: + if (value instanceof byte[]) { + return ByteBuffer.wrap((byte[]) value); + } + return value; + case FIXED: + throw new UnsupportedOperationException("Fixed-precision fields are not yet supported."); + default: + return value; + } + } + /** Converts a Beam {@link Row} to an Iceberg {@link Record}. */ public static Record beamRowToIcebergRecord(org.apache.iceberg.Schema schema, Row row) { if (row.getSchema().getFieldCount() != schema.columns().size()) { diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java index ebb50981bd9d..6eb26e6de4b5 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorterTest.java @@ -18,8 +18,10 @@ package org.apache.beam.sdk.io.iceberg; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -64,7 +66,9 @@ private static byte[] encodeSortKeyHelper(Row row, SortOrder sortOrder) throws E for (int i = 0; i < fields.size(); i++) { columnNames[i] = ICEBERG_SCHEMA.findColumnName(fields.get(i).sourceId()); } - return IcebergRowSorter.encodeSortKey(row, sortOrder, columnNames, ICEBERG_SCHEMA, BEAM_SCHEMA); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + IcebergRowSorter.encodeSortKey(row, sortOrder, columnNames, baos, ICEBERG_SCHEMA, BEAM_SCHEMA); + return baos.toByteArray(); } @Test @@ -86,10 +90,6 @@ public void testStringKeyEncodingOrder() throws Exception { @Test public void testStringCollisionProofing() throws Exception { - // Tests that secondary columns don't bleed into primary columns. - // Row 1: Primary="abc", Secondary="def" - // Row 2: Primary="abcdef", Secondary=null - // In raw byte concatenation, both could equal "abcdef\0" if delimiters or escaping fail. SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("name").asc("value").build(); Row r1 = Row.withSchema(BEAM_SCHEMA).addValues(1, "abc", 1.0, true).build(); @@ -98,7 +98,6 @@ public void testStringCollisionProofing() throws Exception { byte[] k1 = encodeSortKeyHelper(r1, sortOrder); byte[] k2 = encodeSortKeyHelper(r2, sortOrder); - // "abc" must sort lexicographically before "abcdef" assertTrue(BYTE_ARR_COMPARATOR.compare(k1, k2) < 0); } @@ -116,10 +115,7 @@ public void testDescInversion() throws Exception { byte[] k1Desc = encodeSortKeyHelper(r1, sortOrderDesc); byte[] k2Desc = encodeSortKeyHelper(r2, sortOrderDesc); - // Ascending: 10 < 20 assertTrue(BYTE_ARR_COMPARATOR.compare(k1Asc, k2Asc) < 0); - - // Descending: 10 > 20 (inverted bytes) assertTrue(BYTE_ARR_COMPARATOR.compare(k1Desc, k2Desc) > 0); } @@ -182,7 +178,6 @@ public void testEndToEndSorting() { assertEquals(4, sortedList.size()); - // Expected: apple (5) -> banana (2) -> banana (1) -> cherry (10) assertEquals("apple", sortedList.get(0).getString("name")); assertEquals(Integer.valueOf(5), sortedList.get(0).getInt32("id")); @@ -198,8 +193,6 @@ public void testEndToEndSorting() { @Test public void testScaleAndExternalDiskSpill() { - // Verifies sorting operates correctly with thousands of elements, - // proving that BufferedExternalSorter handles memory constraints correctly. SortOrder sortOrder = SortOrder.builderFor(ICEBERG_SCHEMA).asc("id").build(); int count = 5000; @@ -207,7 +200,6 @@ public void testScaleAndExternalDiskSpill() { Random rand = new Random(42); for (int i = 0; i < count; i++) { - // Intentionally insert random IDs to enforce complex sorting int randomId = rand.nextInt(100_000); input.add(Row.withSchema(BEAM_SCHEMA).addValues(randomId, "item" + i, 1.0, true).build()); } @@ -218,13 +210,27 @@ public void testScaleAndExternalDiskSpill() { assertEquals(count, sortedList.size()); - // Validate that the returned dataset is in strictly non-decreasing order of 'id' for (int i = 0; i < sortedList.size() - 1; i++) { int idCurrent = sortedList.get(i).getInt32("id"); int idNext = sortedList.get(i + 1).getInt32("id"); assertTrue( - String.format("Sort violation detected at index %d: %d > %d", i, idCurrent, idNext), + String.format("Sort violation at index %d: %d > %d", i, idCurrent, idNext), idCurrent <= idNext); } } + + @Test + public void testUnsupportedComplexTypeSorting() { + org.apache.iceberg.Schema mapSchema = + new org.apache.iceberg.Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional( + 2, + "attributes", + Types.MapType.ofOptional(3, 4, Types.StringType.get(), Types.StringType.get()))); + + assertThrows( + org.apache.iceberg.exceptions.ValidationException.class, + () -> SortOrder.builderFor(mapSchema).asc("attributes").build()); + } } From 890fe765614d10857cc4f232544d93fa7b21b9b1 Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Fri, 8 May 2026 13:09:41 +0200 Subject: [PATCH 10/21] Set HASH as the default distribution mode in IcebergIO and comprehensively document distribution modes --- .../apache/beam/sdk/io/iceberg/IcebergIO.java | 75 +++++++++++++++++-- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index 7962beb39f14..7b533ae5980a 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -385,7 +385,7 @@ public class IcebergIO { public static WriteRows writeRows(IcebergCatalogConfig catalog) { return new AutoValue_IcebergIO_WriteRows.Builder() .setCatalogConfig(catalog) - .setDistributionMode(DistributionMode.NONE) + .setDistributionMode(DistributionMode.HASH) .setAutoSharding(false) .build(); } @@ -462,18 +462,79 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) { } /** - * Defines distribution of write data. Supported distributions: + * Defines the distribution mode of write data prior to writing. * - *
    - *
  1. {@link DistributionMode.NONE}: don't shuffle rows (default) - *
  2. {@link DistributionMode.HASH}: shuffle rows by partition key before writing data - *
  3. {@link DistributionMode.RANGE}: shuffle rows based on range-partitioning function - *
+ *

The default distribution mode is {@link DistributionMode#HASH}. + * + *

Comparison of Distribution Modes:

+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Comparison of Distribution Modes
ModeDescriptionProsCons
{@link DistributionMode#NONE}No network shuffle is performed. Records are sorted locally on workers prior to writing.Highly lightweight with zero shuffle/network overhead. Best for smaller data volumes.Writers on different workers can write to overlapping min/max key ranges across multiple files. Relies heavily on post-fact compaction.
{@link DistributionMode#HASH}Data is shuffled and consolidated by partition key. All records for a partition are routed to a single worker.Consolidates partition files, eliminating cross-worker file overlapping for partition keys. Excellent worker stability.Can suffer from severe data skew if a single partition contains significantly more data than others (hot partitions).
{@link DistributionMode#RANGE}Data is shuffled based on a user-provided shard/bucket function (e.g., hashing/binning continuous keys).Distributes writes for hot partitions across multiple workers. Eliminates skew while keeping file min/max key ranges tight and non-overlapping.Requires providing a custom {@link SerializableFunction} mapping rows to integer shard/bucket IDs.
+ * + *

Code Samples:

+ * + * 
{@code
+     * // 1. Using default HASH distribution mode (Consolidates by partition key)
+     * pipeline
+     *     .apply(Create.of(BEAM_ROWS))
+     *     .apply(IcebergIO.writeRows(catalogConfig)
+     *         .to(tableId));
+     *
+     * // 2. Using NONE distribution mode (No shuffle, local sorting only)
+     * pipeline
+     *     .apply(Create.of(BEAM_ROWS))
+     *     .apply(IcebergIO.writeRows(catalogConfig)
+     *         .to(tableId)
+     *         .withDistributionMode(DistributionMode.NONE));
+     *
+     * // 3. Using RANGE distribution mode with a custom shard/bucket function to avoid data skew
+     * pipeline
+     *     .apply(Create.of(BEAM_ROWS))
+     *     .apply(IcebergIO.writeRows(catalogConfig)
+     *         .to(tableId)
+     *         .withDistributionMode(DistributionMode.RANGE)
+     *         .withDistributionFunction(row -> {
+     *             // Group timestamps or continuous IDs into 16 parallel, non-overlapping shards
+     *             long timestamp = row.getDateTime("timestamp").getMillis();
+     *             return (int) (Math.abs(timestamp) % 16);
+     *         }));
+     * }
*/ public WriteRows withDistributionMode(DistributionMode mode) { return toBuilder().setDistributionMode(mode).build(); } + /** + * Sets the custom range-distribution function. + * + *

Only applicable when the distribution mode is set to {@link DistributionMode#RANGE}. The + * function maps a Beam {@link Row} to an Integer representing a shard/bucket ID. + */ public WriteRows withDistributionFunction(SerializableFunction shardFn) { return toBuilder().setDistributionFunction(shardFn).build(); } From 0cecd0140f95decbb1cad42cbc44ca0f9a2b29db Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Fri, 8 May 2026 13:12:59 +0200 Subject: [PATCH 11/21] Add testRangeDistribution integration test case for RANGE distribution mode with custom sharding function --- .../sdk/io/iceberg/IcebergIOWriteTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java index c0a18dca0861..294e66be6956 100644 --- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java +++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOWriteTest.java @@ -776,6 +776,51 @@ public void process(@Element KV>> sums) { assertEquals(5L, numWaves); } + @Test + public void testRangeDistribution() { + assumeTrue(distributionMode.equals(HASH_WITH_AUTOSHARDING)); + + Schema schema = Schema.builder().addInt64Field("id").addStringField("name").build(); + + TableIdentifier tableId = + TableIdentifier.of("default", "range_" + Long.toString(UUID.randomUUID().hashCode(), 16)); + Map catalogProps = + ImmutableMap.builder() + .put("type", CatalogUtil.ICEBERG_CATALOG_TYPE_HADOOP) + .put("warehouse", warehouse.location) + .build(); + IcebergCatalogConfig catalog = + IcebergCatalogConfig.builder() + .setCatalogName("name") + .setCatalogProperties(catalogProps) + .build(); + + org.apache.iceberg.Schema icebergSchema = IcebergUtils.beamSchemaToIcebergSchema(schema); + catalog.catalog().createTable(tableId, icebergSchema, PartitionSpec.unpartitioned()); + + PCollection rows = + testPipeline + .apply(GenerateSequence.from(0).to(100)) + .apply( + "Make rows", + MapElements.into(TypeDescriptors.rows()) + .via(i -> Row.withSchema(schema).addValues(i, "name_" + i).build())) + .setRowSchema(schema); + + rows.apply( + "range distribution write", + IcebergIO.writeRows(catalog) + .to(tableId) + .withDistributionMode(DistributionMode.RANGE) + .withDistributionFunction(row -> (int) (row.getInt64("id") % 5))); + + testPipeline.run().waitUntilFinish(); + + Table table = warehouse.loadTable(tableId); + List writtenRecords = ImmutableList.copyOf(IcebergGenerics.read(table).build()); + assertEquals(100, writtenRecords.size()); + } + @Test public void testSortedWrite() { TableIdentifier tableId = From 94fa738a529ef8dd58fc8c4e071dce8f5eb96dbb Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Fri, 8 May 2026 13:20:43 +0200 Subject: [PATCH 12/21] Add Javadoc documentation for withAutosharding method in IcebergIO.java --- .../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index 7b533ae5980a..40ed791a58a6 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -480,7 +480,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) { * {@link DistributionMode#NONE} * No network shuffle is performed. Records are sorted locally on workers prior to writing. * Highly lightweight with zero shuffle/network overhead. Best for smaller data volumes. - * Writers on different workers can write to overlapping min/max key ranges across multiple files. Relies heavily on post-fact compaction. + * Writers on different workers can write to overlapping min/max key ranges across multiple files. Relies heavily on post-fact compaction or query time merges. * * * {@link DistributionMode#HASH} @@ -539,6 +539,18 @@ public WriteRows withDistributionFunction(SerializableFunction sha return toBuilder().setDistributionFunction(shardFn).build(); } + /** + * Enables Beam's dynamic auto-sharding when using {@link DistributionMode#HASH}. + * + *

When enabled, the pipeline uses {@link + * org.apache.beam.sdk.transforms.GroupIntoBatches#withShardedKey()} under the hood. The runner + * (such as Dataflow) dynamically monitors throughput per partition key. If a partition is + * extremely hot, the runner automatically splits it into parallel sub-shards distributed across + * multiple workers to prevent single-worker bottlenecks and out-of-memory (OOM) errors, while + * keeping the number of data files for cold partitions minimal. + * + *

Only applicable when using {@link DistributionMode#HASH}. + */ public WriteRows withAutosharding() { return toBuilder().setAutoSharding(true).build(); } From f24183afc0cd4f1219eedb74031dce5f703616bc Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Fri, 8 May 2026 13:58:59 +0200 Subject: [PATCH 13/21] Update RANGE sharding code sample to use ID partitioning and document auto-sharding overlap limitations --- .../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index 40ed791a58a6..a1eb687ef3bf 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -519,9 +519,9 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) { * .to(tableId) * .withDistributionMode(DistributionMode.RANGE) * .withDistributionFunction(row -> { - * // Group timestamps or continuous IDs into 16 parallel, non-overlapping shards - * long timestamp = row.getDateTime("timestamp").getMillis(); - * return (int) (Math.abs(timestamp) % 16); + * // Group continuous IDs into 16 parallel, non-overlapping shards + * long id = row.getInt64("id"); + * return (int) (id / 10000); * })); * } */ @@ -549,6 +549,11 @@ public WriteRows withDistributionFunction(SerializableFunction sha * multiple workers to prevent single-worker bottlenecks and out-of-memory (OOM) errors, while * keeping the number of data files for cold partitions minimal. * + *

Note that because auto-sharding distributes hot-partition data randomly across worker + * shards, the written data files cannot guarantee non-overlapping key ranges. Downstream + * queries may require read-time sort merges for overlapping file segments until an Iceberg + * compaction job (e.g., `rewriteDataFiles`) is executed. + * *

Only applicable when using {@link DistributionMode#HASH}. */ public WriteRows withAutosharding() { From 670aab1c7dff5033cc9cc49d4eda524db57619cf Mon Sep 17 00:00:00 2001 From: Ale Tognola Date: Fri, 8 May 2026 16:09:14 +0200 Subject: [PATCH 14/21] Document the multi-dimensional distribution mode decision matrix in IcebergIO Javadocs --- .../apache/beam/sdk/io/iceberg/IcebergIO.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index a1eb687ef3bf..29392d7b1013 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -496,6 +496,82 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) { * * * + *

Recommendation Matrix (Sorting & Partitioning vs. Scale):

+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Recommendation Matrix
PartitioningSortingScale / VolumeLatency PriorityRecommended Mode
PartitionedSortedSmallAny{@link DistributionMode#HASH}
PartitionedSortedMedium / LargeLow Write Latency{@link DistributionMode#NONE} (requires post-fact compaction)
PartitionedSortedMedium / LargeLow Read Latency{@link DistributionMode#HASH} with auto-sharding OR {@link DistributionMode#RANGE}
PartitionedUnsortedSmallAny{@link DistributionMode#HASH}
PartitionedUnsortedMedium / LargeAny{@link DistributionMode#HASH} with auto-sharding
UnpartitionedSortedSmallAny{@link DistributionMode#NONE}
UnpartitionedSortedMedium / LargeLow Write Latency{@link DistributionMode#NONE} (requires post-fact compaction)
UnpartitionedSortedMedium / LargeLow Read Latency{@link DistributionMode#RANGE} (with custom sharding function)
UnpartitionedUnsortedAnyAny{@link DistributionMode#NONE}
+ * *

Code Samples:

* * 
{@code

From acf0e560aab0b0d3c47de1fc50a160273bf2c59b Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Fri, 8 May 2026 16:10:58 +0200
Subject: [PATCH 15/21] Add Operational Impact column to distribution modes
 Javadoc table in IcebergIO.java

---
 .../org/apache/beam/sdk/io/iceberg/IcebergIO.java  | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
index 29392d7b1013..64dc517d4c25 100644
--- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
+++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
@@ -506,6 +506,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Scale / Volume
      *     Latency Priority
      *     Recommended Mode
+     *     Operational Impact
      *   
      *   
      *     Partitioned
@@ -513,13 +514,15 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Small
      *     Any
      *     {@link DistributionMode#HASH}
+     *     Consolidates partition files and sorts them locally. Avoids file overlaps for small volumes.
      *   
      *   
      *     Partitioned
      *     Sorted
      *     Medium / Large
      *     Low Write Latency
-     *     {@link DistributionMode#NONE} (requires post-fact compaction)
+     *     {@link DistributionMode#NONE}
+     *     Eliminates shuffle overhead for maximum write speed. Results in overlapping key ranges across files, which requires downstream compaction.
      *   
      *   
      *     Partitioned
@@ -527,6 +530,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Medium / Large
      *     Low Read Latency
      *     {@link DistributionMode#HASH} with auto-sharding OR {@link DistributionMode#RANGE}
+     *     HASH with auto-sharding scales writes for hot partitions but can result in overlapping file ranges requiring query-time sort merges. RANGE sharding distributes hot partitions into sequential, non-overlapping files to optimize reads.
      *   
      *   
      *     Partitioned
@@ -534,6 +538,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Small
      *     Any
      *     {@link DistributionMode#HASH}
+     *     Consolidates data files into single partition directories to prevent file fragmentation.
      *   
      *   
      *     Partitioned
@@ -541,6 +546,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Medium / Large
      *     Any
      *     {@link DistributionMode#HASH} with auto-sharding
+     *     Consolidates partition files while dynamically balancing hot partition writes across parallel workers.
      *   
      *   
      *     Unpartitioned
@@ -548,13 +554,15 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Small
      *     Any
      *     {@link DistributionMode#NONE}
+     *     Bypasses network shuffle for fast, low-volume local sorting.
      *   
      *   
      *     Unpartitioned
      *     Sorted
      *     Medium / Large
      *     Low Write Latency
-     *     {@link DistributionMode#NONE} (requires post-fact compaction)
+     *     {@link DistributionMode#NONE}
+     *     Bypasses network shuffle for parallel worker writes. Requires downstream compaction to resolve overlapping file ranges.
      *   
      *   
      *     Unpartitioned
@@ -562,6 +570,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Medium / Large
      *     Low Read Latency
      *     {@link DistributionMode#RANGE} (with custom sharding function)
+     *     Shards continuous keys into non-overlapping worker ranges. Eliminates single-worker bottlenecks and guarantees zero file overlap for fast queries.
      *   
      *   
      *     Unpartitioned
@@ -569,6 +578,7 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
      *     Any
      *     Any
      *     {@link DistributionMode#NONE}
+     *     Direct, parallel worker writes with maximum throughput and zero network shuffle overhead.
      *   
      * 
      *

From e22d4dc11b1805a62b4fd2c792f57983e39b017e Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Fri, 8 May 2026 18:01:37 +0200
Subject: [PATCH 16/21] Fix ClassCastException in IcebergUtils string field
 copying by using value-level toString conversion

---
 .../java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
index 0e8def1656d5..7f48a0d0128c 100644
--- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
+++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
@@ -390,7 +390,10 @@ private static void copyFieldIntoRecord(Record rec, Types.NestedField field, Row
         rec.setField(name, getIcebergTimestampValue(val, ts.shouldAdjustToUTC()));
         break;
       case STRING:
-        Optional.ofNullable(value.getString(name)).ifPresent(v -> rec.setField(name, v));
+        Object strVal = value.getValue(name);
+        if (strVal != null) {
+          rec.setField(name, strVal.toString());
+        }
         break;
       case UUID:
         Optional.ofNullable(value.getBytes(name))

From 1ab4fcf35acae39f56a90e9e07ab518754112ce1 Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Fri, 8 May 2026 18:06:24 +0200
Subject: [PATCH 17/21] Add comprehensive test scenarios for dynamic BigDecimal
 and Integer to String conversions in IcebergUtilsTest.java

---
 .../apache/beam/sdk/io/iceberg/IcebergUtilsTest.java | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
index c9026522dba3..d62aab0d3d6f 100644
--- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
+++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
@@ -269,6 +269,18 @@ public void testMapOfRecords() {
           IcebergUtils.beamRowToIcebergRecord(RECORD_MAP_ICEBERG_SCHEMA, ROW_MAP_OF_ROWS);
       assertEquals(RECORD_MAP_OF_RECORDS, actual);
     }
+
+    @Test
+    public void testBigDecimalToStringConversion() {
+      BigDecimal num = new BigDecimal("987654321.123456789");
+      checkRowValueToRecordValue(
+          Schema.FieldType.DECIMAL, num, Types.StringType.get(), "987654321.123456789");
+    }
+
+    @Test
+    public void testIntegerToStringConversion() {
+      checkRowValueToRecordValue(Schema.FieldType.INT32, 42, Types.StringType.get(), "42");
+    }
   }
 
   @RunWith(JUnit4.class)

From 1e191808d45810bf31bcf013ef5e8d802167ae61 Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Fri, 8 May 2026 18:07:51 +0200
Subject: [PATCH 18/21] Add thorough test scenarios for Double, Boolean, and
 Null value to String conversions in IcebergUtilsTest.java

---
 .../beam/sdk/io/iceberg/IcebergUtilsTest.java | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
index d62aab0d3d6f..d6b2bf11370e 100644
--- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
+++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
@@ -281,6 +281,30 @@ public void testBigDecimalToStringConversion() {
     public void testIntegerToStringConversion() {
       checkRowValueToRecordValue(Schema.FieldType.INT32, 42, Types.StringType.get(), "42");
     }
+
+    @Test
+    public void testDoubleToStringConversion() {
+      checkRowValueToRecordValue(
+          Schema.FieldType.DOUBLE, 3.14159, Types.StringType.get(), "3.14159");
+    }
+
+    @Test
+    public void testBooleanToStringConversion() {
+      checkRowValueToRecordValue(Schema.FieldType.BOOLEAN, true, Types.StringType.get(), "true");
+    }
+
+    @Test
+    public void testNullStringConversion() {
+      Schema beamSchema =
+          Schema.of(Schema.Field.of("v", Schema.FieldType.STRING).withNullable(true));
+      Row row = Row.withSchema(beamSchema).addValue(null).build();
+
+      org.apache.iceberg.Schema icebergSchema =
+          new org.apache.iceberg.Schema(optional(0, "v", Types.StringType.get()));
+      Record record = IcebergUtils.beamRowToIcebergRecord(icebergSchema, row);
+
+      assertEquals(null, record.getField("v"));
+    }
   }
 
   @RunWith(JUnit4.class)

From 4d20b1180ab6743ded67a763c6ad638ac14b4ef1 Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Fri, 8 May 2026 18:17:59 +0200
Subject: [PATCH 19/21] Use catalog.buildTable to set SortOrder during dynamic
 table creation in WritePartitionedRowsToFiles.java

---
 .../io/iceberg/WritePartitionedRowsToFiles.java    | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java
index 92c20433c462..0da0d4c5968c 100644
--- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java
+++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java
@@ -44,6 +44,7 @@
 import org.apache.iceberg.PartitionField;
 import org.apache.iceberg.PartitionKey;
 import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.SortOrder;
 import org.apache.iceberg.StructLike;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.catalog.Catalog;
@@ -243,14 +244,23 @@ LastRefreshedTable getOrCreateTable(IcebergDestination destination, Schema dataS
         } catch (NoSuchTableException e) { // Otherwise, create the table
           org.apache.iceberg.Schema tableSchema =
               IcebergUtils.beamSchemaToIcebergSchema(dataSchema);
+          SortOrder sortOrder =
+              createConfig != null ? createConfig.getSortOrder() : SortOrder.unsorted();
           try {
-            table = catalog.createTable(identifier, tableSchema, partitionSpec, tableProperties);
+            table =
+                catalog
+                    .buildTable(identifier, tableSchema)
+                    .withPartitionSpec(partitionSpec)
+                    .withSortOrder(sortOrder)
+                    .withProperties(tableProperties)
+                    .create();
             LOG.info(
                 "Created Iceberg table '{}' with schema: {}\n"
-                    + ", partition spec: {}, table properties: {}",
+                    + ", partition spec: {}, sort order: {}, table properties: {}",
                 identifier,
                 tableSchema,
                 partitionSpec,
+                sortOrder,
                 tableProperties);
           } catch (AlreadyExistsException ignored) {
             // race condition: another worker already created this table

From fca10f87b3ae31bf16108a550968302465899211 Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Sat, 9 May 2026 08:47:41 +0200
Subject: [PATCH 20/21] Adding a note and a warning

---
 .../org/apache/beam/sdk/io/iceberg/IcebergIO.java     | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
index 64dc517d4c25..9b18a77326e0 100644
--- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
+++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
@@ -462,9 +462,16 @@ public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
     }
 
     /**
-     * Defines the distribution mode of write data prior to writing.
+     * The default distribution mode is {@link DistributionMode#HASH}.
      *
-     * 
The default distribution mode is {@link DistributionMode#HASH}.
+     * 
Warning on HASH mode: Utilizing {@code HASH} distribution mode (with or without
+     * auto-sharding) can suffer from large unpartitioned or skewed writes if key spaces are
+     * not uniformly distributed. This can bottleneck workers and produce fragmented layout files.
+     *
+     * 
Note on RANGE mode: When utilizing {@code RANGE} distribution mode, it is
+     * recommended that the custom distribution function is designed to produce adequately sized and
+     * strictly non-overlapping ranges of the sorting column to optimize downstream read
+     * performance.
      *
      * 
Comparison of Distribution Modes:

      *

From e576dbff26a62d866bdaa8c2266f09d672b3a667 Mon Sep 17 00:00:00 2001
From: Ale Tognola 
Date: Tue, 12 May 2026 16:05:25 +0200
Subject: [PATCH 21/21] Adding scratch files

---
 scratch/iceberg-scale-test/build.gradle       |  43 +
 .../gradle/wrapper/gradle-wrapper.jar         | Bin 0 -> 63721 bytes
 .../gradle/wrapper/gradle-wrapper.properties  |   7 +
 scratch/iceberg-scale-test/gradlew            | 249 +++++
 scratch/iceberg-scale-test/settings.gradle    |   1 +
 .../AssignDestinationsAndPartitions.java      | 183 ++++
 .../apache/beam/sdk/io/iceberg/IcebergIO.java | 887 ++++++++++++++++++
 .../beam/sdk/io/iceberg/IcebergRowSorter.java | 269 ++++++
 .../beam/sdk/io/iceberg/IcebergUtils.java     | 683 ++++++++++++++
 .../iceberg/WritePartitionedRowsToFiles.java  | 276 ++++++
 .../test/IcebergBigQueryScaleTest.java        | 116 +++
 11 files changed, 2714 insertions(+)
 create mode 100644 scratch/iceberg-scale-test/build.gradle
 create mode 100644 scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.jar
 create mode 100644 scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.properties
 create mode 100755 scratch/iceberg-scale-test/gradlew
 create mode 100644 scratch/iceberg-scale-test/settings.gradle
 create mode 100644 scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java
 create mode 100644 scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
 create mode 100644 scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java
 create mode 100644 scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
 create mode 100644 scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java
 create mode 100644 scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/test/IcebergBigQueryScaleTest.java

diff --git a/scratch/iceberg-scale-test/build.gradle b/scratch/iceberg-scale-test/build.gradle
new file mode 100644
index 000000000000..0ab76e165b64
--- /dev/null
+++ b/scratch/iceberg-scale-test/build.gradle
@@ -0,0 +1,43 @@
+plugins {
+    id 'java'
+    id 'application'
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url "https://repository.apache.org/snapshots/"
+    }
+}
+
+dependencies {
+    implementation "org.apache.beam:beam-sdks-java-core:2.74.0-SNAPSHOT"
+    implementation "org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.74.0-SNAPSHOT"
+    implementation "org.apache.beam:beam-sdks-java-io-iceberg:2.74.0-SNAPSHOT"
+    implementation "org.apache.beam:beam-sdks-java-extensions-sorter:2.74.0-SNAPSHOT"
+    implementation "org.apache.beam:beam-runners-google-cloud-dataflow-java:2.74.0-SNAPSHOT"
+    
+    implementation "org.apache.iceberg:iceberg-core:1.4.3"
+    implementation "org.apache.iceberg:iceberg-api:1.4.3"
+    implementation "org.apache.iceberg:iceberg-data:1.4.3"
+    implementation "org.apache.iceberg:iceberg-gcp:1.4.3"
+    implementation "com.google.cloud.bigdataoss:gcs-connector:hadoop2-2.2.26"
+    implementation "org.apache.hadoop:hadoop-client:3.4.2"
+    implementation "org.apache.hadoop:hadoop-common:3.4.2"
+    
+    implementation "org.slf4j:slf4j-api:1.7.30"
+    implementation "org.slf4j:slf4j-jdk14:1.7.30"
+    
+    annotationProcessor "com.google.auto.value:auto-value:1.9"
+    compileOnly "com.google.auto.value:auto-value-annotations:1.9"
+}
+
+application {
+    mainClass = 'org.apache.beam.sdk.io.iceberg.test.IcebergBigQueryScaleTest'
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+}
diff --git a/scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.jar b/scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 0000000000000000000000000000000000000000..7f93135c49b765f8051ef9d0a6055ff8e46073d8
GIT binary patch
literal 63721
zcmb5Wb9gP!wgnp7wrv|bwr$&XvSZt}Z6`anZSUAlc9NHKf9JdJ;NJVr`=eI(_pMp0
zy1VAAG3FfAOI`{X1O)&90s;U4K;XLp008~hCjbEC_fbYfS%6kTR+JtXK>nW$ZR+`W
ze|#J8f4A@M|F5BpfUJb5h>|j$jOe}0oE!`Zf6fM>CR?!y@zU(cL8NsKk`a
z6tx5mAkdjD;J=LcJ;;Aw8p!v#ouk>mUDZF@
zK>yvw%+bKu+T{Nk@LZ;zkYy0HBKw06_IWcMHo*0HKpTsEFZhn5qCHH9j
z)|XpN&{`!0a>Vl+PmdQc)Yg4A(AG-z!+@Q#eHr&g<9D?7E)_aEB?s_rx>UE9TUq|?
z;(ggJt>9l?C|zoO@5)tu?EV0x_7T17q4fF-q3{yZ^ipUbKcRZ4Qftd!xO(#UGhb2y>?*@{xq%`(-`2T^vc=#<
zx!+@4pRdk&*1ht2OWk^Z5IAQ0YTAXLkL{(D*$gENaD)7A%^XXrCchN&z2x+*>o2FwPFjWpeaL=!tzv#JOW#(
z$B)Nel<+$bkH1KZv3&-}=SiG~w2sbDbAWarg%5>YbC|}*d9hBjBkR(@tyM0T)FO$#
zPtRXukGPnOd)~z=?avu+4Co@wF}1T)-uh5jI<1$HLtyDrVak{gw`mcH@Q-@wg{v^c
zRzu}hMKFHV<8w}o*yg6p@Sq%=gkd~;`_VGTS?L@yVu`xuGy+dH6YOwcP6ZE`_0rK%
zAx5!FjDuss`FQ3eF|mhrWkjux(Pny^k$u_)dyCSEbAsecHsq#8B3n3kDU(zW5yE|(
zgc>sFQywFj5}U*qtF9Y(bi*;>B7WJykcAXF86@)z|0-Vm@jt!EPoLA6>r)?@DIobIZ5Sx
zsc@OC{b|3%vaMbyeM|O^UxEYlEMHK4r)V-{r)_yz`w1*xV0|lh-LQOP`OP`Pk1aW(
z8DSlGN>Ts|n*xj+%If~+E_BxK)~5T#w6Q1WEKt{!Xtbd`J;`2a>8boRo;7u2M&iOop4qcy<)z023=oghSFV
zST;?S;ye+dRQe>ygiJ6HCv4;~3DHtJ({fWeE~$H@mKn@Oh6Z(_sO>01JwH5oA4nvK
zr5Sr^g+LC
zLt(i&ecdmqsIJGNOSUyUpglvhhrY8lGkzO=0USEKNL%8zHshS>Qziu|`eyWP^5xL4
zRP122_dCJl>hZc~?58w~>`P_s18VoU|7(|Eit0-lZRgLTZKNq5{k
zE?V=`7=R&ro(X%LTS*f+#H-mGo_j3dm@F_krAYegDLk6UV{`UKE;{YSsn$
z(yz{v1@p|p!0>g04!eRSrSVb>MQYPr8_MA|MpoGzqyd*$@4j|)cD_%^Hrd>SorF>@
zBX+V<@vEB5PRLGR(uP9&U&5=(HVc?6B58NJT_igiAH*q~Wb`dDZpJSKfy5#Aag4IX
zj~uv74EQ_Q_1qaXWI!7Vf@ZrdUhZFE;L&P_Xr8l@GMkhc#=plV0+g(ki>+7fO%?Jb
zl+bTy7q{w^pTb{>(Xf2q1BVdq?#f=!geqssXp
z4pMu*q;iiHmA*IjOj4`4S&|8@gSw*^{|PT}Aw~}ZXU`6=vZB=GGeMm}V6W46|pU&58~P+?LUs%n@J}CSrICkeng6YJ^M?
zS(W?K4nOtoBe4tvBXs@@`i?4G$S2W&;$z8VBSM;Mn9
zxcaEiQ9=vS|bIJ>*tf9AH~m&U%2+Dim<)E=}KORp+cZ^!@wI`h1NVBXu{@%hB2Cq(dXx_aQ9x3mr*fwL5!ZryQqi|KFJuzvP
zK1)nrKZ7U+B{1ZmJub?4)Ln^J6k!i0t~VO#=q1{?T)%OV?MN}k5M{}vjyZu#M0_*u
z8jwZKJ#Df~1jcLXZL7bnCEhB6IzQZ-GcoQJ!16I*39iazoVGugcKA{lhiHg4Ta2fD
zk1Utyc5%QzZ$s3;p0N+N8VX{sd!~l*Ta3|t>lhI&G`sr6L~G5Lul`>m
z{!^INm?J|&7X=;{XveF!(b*=?9NAp4y&r&N3(GKcW4rS(Ejk|Lzs1PrxPI_owB-`H
zg3(Rruh^&)`TKA6+_!n>RdI6pw>Vt1_j&+bKIaMTYLiqhZ#y_=J8`TK{Jd<7l9&sY
z^^`hmi7^14s16B6)1O;vJWOF$=$B5ONW;;2&|pUvJlmeUS&F;DbSHCrEb0QBDR|my
zIs+pE0Y^`qJTyH-_mP=)Y+u^LHcuZhsM3+P||?+W#V!_6E-8boP#R-*na4!o-Q1
zVthtYhK{mDhF(&7Okzo9dTi03X(AE{8cH$JIg%MEQca`S
zy@8{Fjft~~BdzWC(di#X{ny;!yYGK9b@=b|zcKZ{vv4D8i+`ilOPl;PJl{!&5-0!w
z^fOl#|}vVg%=n)@_e1BrP)`A
zKPgs`O0EO}Y2KWLuo`iGaKu1k#YR6BMySxQf2V++Wo{6EHmK>A~Q5o73yM
z-RbxC7Qdh0Cz!nG+7BRZE>~FLI-?&W_rJUl-8FDIaXoNBL)@1hwKa^wOr1($*5h~T
zF;%f^%<$p8Y_yu(JEg=c_O!aZ#)Gjh$n(hfJAp$C2he555W5zdrBqjFmo|VY+el;o
z=*D_w|GXG|p0**hQ7~9-n|y5k%B}TAF0iarDM!q-jYbR^us(>&y;n^2l0C%@2B}KM
zyeRT9)oMt97Agvc4sEKUEy%MpXr2vz*lb
zh*L}}iG>-pqDRw7ud{=FvTD?}xjD)w{`KzjNom-$jS^;iw0+7nXSnt1R@G|VqoRhE%12nm+PH?9`(4rM0kfrZzIK9JU=^$YNyLvAIoxl#Q)xxDz!^0@zZ
zSCs$nfcxK_vRYM34O<1}QHZ|hp4`ioX3x8(UV(FU$J@o%tw3t4k1QPmlEpZa2IujG&(roX_q*%e`Hq|);0;@k
z0z=fZiFckp#JzW0p+2A+D$PC~IsakhJJkG(c;CqAgFfU0Z`u$PzG~-9I1oPHrCw&)@s^Dc~^)#HPW0Ra}J^=|h7Fs*<8|b13ZzG6MP*Q1dkoZ6&A^!}|hbjM{2HpqlSXv_UUg1U4gn
z3Q)2VjU^ti1myodv+tjhSZp%D978m~p&
z43uZUrraHs80Mq&vcetqfQpQP?m!CFj)44t8Z}k`E798wxg&~aCm+DBoI+nKq}&j^
zlPY3W$)K;KtEajks1`G?-@me7C>{PiiBu+41#yU_c(dITaqE?IQ(DBu+c^Ux!>pCj
zLC|HJGU*v+!it1(;3e`6igkH(VA)-S+k(*yqxMgUah3$@C
zz`7hEM47xr>j8^g`%*f=6S5n>z%Bt_Fg{Tvmr+MIsCx=0gsu_sF`q2hlkEmisz#Fy
zj_0;zUWr;Gz}$BS%Y`meb(=$d%@Crs(OoJ|}m#<7=-A~PQbyN$x%2iXP2@e*nO0b7AwfH8cCUa*Wfu@b)D_>I*%uE4O3
z(lfnB`-Xf*LfC)E}e?%X2kK7DItK6Tf<+M^mX0Ijf_!IP>7c8IZX%8_#0060P{QMuV^B9i<^E`_Qf0pv9(P%_s8D`qvDE9LK9u-jB}J2S`(mCO&XHTS04Z5Ez*vl^T%!^$~EH8M-UdwhegL>3IQ*)(MtuH2Xt1p!fS4o~*rR?WLxlA!sjc2(O
znjJn~wQ!Fp9s2e^IWP1C<4%sFF}T4omr}7+4asciyo3DntTgWIzhQpQirM$9{EbQd
z3jz9vS@{aOqTQHI|l#aUV@2Q^Wko4T0T04Me4!2nsdrA8QY1%fnAYb~d2GDz@lAtfcHq(P7
zaMBAGo}+NcE-K*@9y;Vt3*(aCaMKXBB*BJcD_Qnxpt75r?GeAQ}*|>pYJE=uZb73
zC>sv)18)q#EGrTG6io*}JLuB_jP3AU1Uiu$D7r|2_zlIGb9
zjhst#ni)Y`$)!fc#reM*$~iaYoz~_Cy7J3ZTiPm)E?%`fbk`3Tu-F#`{i!l5pNEn5
zO-Tw-=TojYhzT{J=?SZj=Z8#|eoF>434b-DXiUsignxXNaR3
zm_}4iWU$gt2Mw5NvZ5(VpF`?X*f2UZDs1TEa1oZCif?Jdgr{>O~7}-$|BZ7I(IKW`{f;@|IZFX*R8&iT=
zoWstN8&R;}@2Ka%d3vrLtR|O??ben;k8QbS-WB0VgiCz;<$pBmIZdN!aalyCSEm)crpS9dcD^Y@XT1a3+zpi-`D}e#HV<}
z$Y(G&o~PvL-xSVD5D?JqF3?B9rxGWeb=oEGJ3vRp5xfBPlngh1O$yI95EL+T8{GC@
z98i1H9KhZGFl|;`)_=QpM6H?eDPpw~^(aFQWwyXZ8_EEE4#@QeT_URray*mEOGsGc
z6|sdXtq!hVZo=d#+9^@lm&L5|q&-GDCyUx#YQiccq;spOBe3V+VKdjJA=IL=Zn%P}
zNk=_8u}VhzFf{UYZV0`lUwcD&)9AFx0@Fc6LD9A6Rd1=ga>Mi0)_QxM2ddCVRmZ0d
z+J=uXc(?5JLX3=)e)Jm$HS2yF`44IKhwRnm2*669_J=2LlwuF5$1tAo@ROSU@-y+;Foy2IEl2^V1N;fk~YR
z?&EP8#t&m0B=?aJeuz~lHjAzRBX>&x=A;gIvb>MD{XEV
zV%l-+9N-)i;YH%nKP?>f`=?#`>B(`*t`aiPLoQM(a6(qs4p5KFjDBN?8JGrf3z8>=
zi7sD)c)Nm~x{e<^jy4nTx${P~cwz_*a>%0_;ULou3kHCAD7EYkw@l$8TN#LO9jC(
z1BeFW`k+bu5e8Ns^a8dPcjEVHM;r6UX+cN=Uy7HU)j-myRU0wHd$A1fNI~`4;I~`zC)3ul#8#^rXVSO*m}Ag>c%_;nj=Nv$rCZ
z*~L@C@OZg%Q^m)lc-kcX&a*a5`y&DaRxh6O*dfhLfF+fU5wKs(1v*!TkZidw*)YBP
za@r`3+^IHRFeO%!ai%rxy;R;;V^Fr=OJlpBX;(b*3+SIw}7=
zIq$*Thr(Zft-RlY)D3e8V;BmD&HOfX+E$H#Y@B3?UL5L~_fA-@*IB-!gItK7PIgG9
zgWuGZK_nuZjHVT_Fv(XxtU%)58;W39vzTI2n&)&4Dmq7&JX6G>XFaAR{7_3QB6zsT
z?$L8c*WdN~nZGiscY%5KljQARN;`w$gho=p006z;n(qIQ*Zu<``TMO3n0{ARL@gYh
zoRwS*|Niw~cR!?hE{m*y@F`1)vx-JRfqET=dJ5_(076st(=lFfjtKHoYg`k3oNmo_
zNbQEw8&sO5jAYmkD|Zaz_yUb0rC})U!rCHOl}JhbYIDLzLvrZVw0~JO`d*6f;X&?V=#T@ND*cv^I;`sFeq4
z##H5;gpZTb^0Hz@3C*~u0AqqNZ-r%rN3KD~%Gw`0XsIq$(^MEb<~H(2*5G^<2(*aI
z%7}WB+TRlMIrEK#s0
z93xn*Ohb=kWFc)BNHG4I(~RPn-R8#0lqyBBz5OM6o5|>x9LK@%HaM}}Y5goCQRt2C
z{j*2TtT4ne!Z}vh89mjwiSXG=%DURar~=kGNNaO_+Nkb+tRi~Rkf!7a$*QlavziD(
z83s4GmQ^Wf*0Bd04f#0HX@ua_d8
z23~z*53ePD6@xwZ(vdl0DLc=>cPIOPOdca&MyR^jhhKrdQO?_jJh`xV3GKz&2lvP8
zEOwW6L*ufvK;TN{=S&R@pzV^U=QNk^Ec}5H
z+2~JvEVA{`uMAr)?Kf|aW>33`)UL@bnfIUQc~L;TsTQ6>r-<^rB8uoNOJ>HWgqMI8
zSW}pZmp_;z_2O5_RD|fGyTxaxk53Hg_3Khc<8AUzV|ZeK{fp|Ne933=1&_^Dbv5^u
zB9n=*)k*tjHDRJ@$bp9mrh}qFn*s}npMl5BMDC%Hs0M0g-hW~P*3CNG06G!MOPEQ_
zi}Qs-6M8aMt;sL$vlmVBR^+Ry<64jrm1EI1%#j?c?4b*7>)a{aDw#TfTYKq+SjEFA
z(aJ&z_0?0JB83D-i3Vh+o|XV4UP+YJ$9Boid2^M2en@APw&wx7vU~t$r2V`F|7Qfo
z>WKgI@eNBZ-+Og<{u2ZiG%>YvH2L3fNpV9J;WLJoBZda)01Rn;o@){01{7E#ke(7U
zHK>S#qZ(N=aoae*4X!0A{)nu0R_sKpi1{)u>GVjC+b5Jyl6#AoQ-1_3UDovNSo`T>
z?c-@7XX*2GMy?k?{g)7?Sv;SJkmxYPJPs!&QqB12ejq`Lee^-cDveVWL^CTUldb(G
zjDGe(O4P=S{4fF=#~oAu>LG>wrU^z_?3yt24FOx>}{^lCGh8?vtvY$^hbZ)9I0E3r3NOlb9I?F-Yc=r$*~l`4N^xzlV~N
zl~#oc>U)Yjl0BxV>O*Kr@lKT{Z09OXt2GlvE38nfs+DD7exl|&vT;)>VFXJVZp9Np
zDK}aO;R3~ag$X*|hRVY3OPax|PG`@_ESc8E!mHRByJbZQRS38V2F__7MW~sgh!a>98Q2%lUNFO=^xU52|?D=IK#QjwBky-C>zOWlsiiM&1n
z;!&1((Xn1$9K}xabq~222gYvx3hnZPg}VMF_GV~5ocE=-v>V=T&RsLBo&`)DOyIj*
zLV{h)JU_y*7SdRtDajP_Y+rBkNN*1_TXiKwHH2&p51d(#zv~s#HwbNy?<+(=9WBvo
zw2hkk2Dj%kTFhY+$T+W-b7@qD!bkfN#Z2ng@Pd=i3-i?xYfs5Z*1hO?kd7Sp^9`;Y
zM2jeGg<-nJD1er@Pc_cSY7wo5dzQX44=%6rn}P_SRbpzsA{6B+!$3B0#;}qwO37G^
zL(V_5JK`XT?OHVk|{_$vQ|oNEpab*BO4F
zUTNQ7RUhnRsU`TK#~`)$icsvKh~(pl=3p6m98@k3P#~upd=k*u20SNcb{l^1rUa)>qO997)pYRWMncC8A&&MHlbW?7i^7M`+B$hH~Y|J
zd>FYOGQ;j>Zc2e7R{KK7)0>>nn_jYJy&o@sK!4G>-rLKM8Hv)f;hi1D2fAc$+six2
zyVZ@wZ6x|fJ!4KrpCJY=!Mq0;)X)OoS~{Lkh6u8J`eK%u0WtKh6B>GW_)PVc
zl}-k`p09qwGtZ@VbYJC!>29V?Dr>>vk?)o(x?!z*9DJ||9qG-&G~#kXxbw{KKYy}J
zQKa-dPt~M~E}V?PhW0R26xdA%1T*%ra6SguGu50YHngOTIv)@N|YttEXo#OZfgtP7;H?EeZZxo<}3YlYxtBq
znJ!WFR^tmGf0Py}N?kZ(#=VtpC@%xJkDmfcCoBTxq
zr_|5gP?u1@vJZbxPZ|G0AW4=tpb84gM2DpJU||(b8kMOV1S3|(yuwZJ&rIiFW(U;5
zUtAW`O6F6Zy+eZ1EDuP~AAHlSY-+A_eI5Gx)%*uro5tljy}kCZU*_d7)oJ>oQSZ3*
zneTn`{gnNC&uJd)0aMBzAg021?YJ~b(fmkwZAd696a=0NzBAqBN54KuNDwa*no(^O
z6p05bioXUR^uXjpTol*ppHp%1v9e)vkoUAUJyBx3lw0UO39b0?^{}yb!$yca(@DUn
zCquRF?t=Zb9`Ed3AI6|L{eX~ijVH`VzSMheKoP7LSSf4g>md>`yi!TkoG5P>Ofp+n
z(v~rW+(5L96L{vBb^g51B=(o)?%%xhvT*A5btOpw(TKh^g^4c
zw>0%X!_0`{iN%RbVk+A^f{w-4-SSf*fu@FhruNL##F~sF24O~u
zyYF<3el2b$$wZ_|uW#@Ak+VAGk#e|kS8nL1g>2B-SNMjMp^8;-FfeofY2fphFHO!{
z*!o4oTb{4e;S<|JEs<1_hPsmAlVNk?_5-Fp5KKU&d#FiNW~Y+pVFk@Cua1I{T+1|+
zHx6rFMor)7L)krbilqsWwy@T+g3DiH5MyVf8Wy}XbEaoFIDr~y;@r&I>FMW{
z?Q+(IgyebZ)-i4jNoXQhq4Muy9Fv+OxU;9_Jmn+<`mEC#%2Q_2bpcgzcinygNI!&^
z=V$)o2&Yz04~+&pPWWn`rrWxJ&}8khR)6B(--!9Q
zubo}h+1T)>a@c)H^i``@<^j?|r4*{;tQf78(xn0g39IoZw0(CwY1f<%F>kEaJ
zp9u|IeMY5mRdAlw*+gSN^5$Q)ShM<~E=(c8QM+T-Qk)FyKz#Sw0EJ*edYcuOtO#~Cx^(M7w5
z3)rl#L)rF|(Vun2LkFr!rg8Q@=r>9p>(t3Gf_auiJ2Xx9HmxYTa|=MH_SUlYL`mz9
zTTS$`%;D-|Jt}AP1&k7PcnfFNTH0A-*FmxstjBDiZX?}%u%Yq94$fUT&z6od+(Uk>
zuqsld#G(b$G8tus=M!N#oPd|PVFX)?M?tCD0tS%2IGTfh}3YA3f&UM)W$_GNV8
zQo+a(ml2Km4o6O%gKTCSDNq+#zCTIQ1*`TIJh~k6Gp;htHBFnne))rlFdGqwC6dx2+La1&Mnko*352k0y
z+tQcwndQlX`nc6nb$A9?<-o|r*%aWXV#=6PQic0Ok_D;q>wbv&j7cKc!w4~KF#-{6
z(S%6Za)WpGIWf7jZ3svNG5OLs0>vCL9{V7cgO%zevIVMH{WgP*^D9ws&OqA{yr|m|
zKD4*07dGXshJHd#e%x%J+qmS^lS|0Bp?{drv;{@{l9ArPO&?Q5=?OO9=}h$oVe#3b
z3Yofj&Cb}WC$PxmRRS)H%&$1-)z7jELS}!u!zQ?A^Y{Tv4QVt*vd@uj-^t2fYRzQj
zfxGR>-q|o$3sGn^#VzZ!QQx?h9`njeJry}@x?|k0-GTTA4y3t2E`3DZ!A~D?GiJup
z)8%PK2^9OVRlP(24P^4_<|D=H^7}WlWu#LgsdHzB%cPy|f8dD3|A^mh4WXxhLTVu_
z@abE{6Saz|Y{rXYPd4$tfPYo}ef(oQWZ=4Bct-=_9`#Qgp4ma$n$`tOwq#&E18$B;
z@Bp)bn3&rEi0>fWWZ@7k5WazfoX`SCO4jQWwVuo+$PmSZn^Hz?O(-tW@*DGxuf)V1
zO_xm&;NVCaHD4dqt(-MlszI3F-p?0!-e$fbiCeuaw66h^TTDLWuaV<@C-`=Xe5WL)
zwooG7h>4&*)p3pKMS3O!4>-4jQUN}iAMQ)2*70?hP~)TzzR?-f@?Aqy$$1Iy8VGG$
zMM?8;j!pUX7QQD$gRc_#+=raAS577ga-w?jd`vCiN5lu)dEUkkUPl9!?{$IJNxQys
z*E4e$eF&n&+AMRQR2gcaFEjAy*r)G!s(P6D&TfoApMFC_*Ftx0|D0@E-=B7tezU@d
zZ{hGiN;YLIoSeRS;9o%dEua4b%4R3;$SugDjP$x;Z!M!@QibuSBb)HY!3zJ7M;^jw
zlx6AD50FD&p3JyP*>o+t9YWW8(7P2t!VQQ21pHJOcG_SXQD;(5aX#M6x##5H_Re>6lPyDCjxr*R(+HE%c&QN+b^tbT
zXBJk?p)zhJj#I?&Y2n&~XiytG9!1ox;bw5Rbj~)7c(MFBb4>IiRATdhg
zmiEFlj@S_hwYYI(ki{}&<;_7(Z0Qkfq>am
z&LtL=2qc7rWguk3BtE4zL41@#S;NN*-jWw|7Kx7H7~_%7fPt;TIX}Ubo>;Rmj94V>
zNB1=;-9AR7s`Pxn}t_6^3ahlq53e&!Lh85uG
zec0vJY_6e`tg7LgfrJ3k!DjR)Bi#L@DHIrZ`sK=<5O0Ip!fxGf*OgGSpP@Hbbe&$9
z;ZI}8lEoC2_7;%L2=w?tb%1oL0V+=Z`7b=P&lNGY;yVBazXRYu;+cQDKvm*7NCxu&i;zub
zAJh#11%?w>E2rf2e~C4+rAb-&$^vsdACs7
z@|Ra!OfVM(ke{vyiqh7puf&Yp6cd6{DptUteYfIRWG3pI+5<
zBVBI_xkBAc<(pcb$!Y%dTW(b;B;2pOI-(QCsLv@U-D1XJ
z(Gk8Q3l7Ws46Aktuj>|s{$6zA&xCPuXL-kB`CgYMs}4IeyG*P51IDwW?8UNQd+$i~
zlxOPtSi5L|gJcF@DwmJA5Ju8HEJ>o{{upwIpb!f{2(vLNBw`7xMbvcw<^{Fj@E~1(
z?w`iIMieunS#>nXlmUcSMU+D3rX28f?s7z;X=se6bo8;5vM|O^(D6{A9*ChnGH!RG
zP##3>LDC3jZPE4PH32AxrqPk|yIIrq~`aL-=}`okhNu9aT%q
z1b)7iJ)CN=V#Ly84N_r7U^SH2FGdE5FpTO2
z630TF$P>GNMu8`rOytb(lB2};`;P4YNwW1<5d3Q~AX#P0aX}R2b2)`rgkp#zTxcGj
zAV^cvFbhP|JgWrq_e`~exr~sIR$6p5V?o4Wym3kQ3HA+;Pr$bQ0(PmADVO%MKL!^q
z?zAM8j1l4jrq|5X+V!8S*2Wl@=7*pPgciTVK6kS1Ge
zMsd_u6DFK$jTnvVtE;qa+8(1sGBu~n&F%dh(&c(Zs4Fc#A=gG^^%^AyH}1^?|8quj
zl@Z47h$){PlELJgYZCIHHL=
z{U8O>Tw4x3<1{?$8>k-P<}1y9DmAZP_;(3Y*{Sk^H^A=_iSJ@+s5ktgwTXz_2$~W9>VVZsfwCm@s0sQ
zeB50_yu@uS+e7QoPvdCwDz{prjo(AFwR%C?z`EL{1`|coJHQTk^nX=tvs1<0arUOJ
z!^`*x&&BvTYmemyZ)2p~{%eYX=JVR?DYr(rNgqRMA5E1PR1Iw=prk=L2ldy3r3Vg@27IZx43+ywyzr-X*p*d@tZV+!U#~$-q=8c
zgdSuh#r?b4GhEGNai)ayHQpk>5(%j5c@C1K3(W1pb~HeHpaqijJZa-e6vq_8t-^M^
zBJxq|MqZc?pjXPIH}70a5vt!IUh;l}<>VX<-Qcv^u@5(@@M2CHSe_hD$VG-eiV^V(
zj7*9T0?di?P$FaD6oo?)<)QT>Npf6Og!GO^GmPV(Km0!=+dE&bk#SNI+C9RGQ|{~O*VC+tXK3!n`5
zHfl6>lwf_aEVV3`0T!aHNZLsj$paS$=LL(?b!Czaa5bbSuZ6#$_@LK<(7yrrl+80|
z{tOFd=|ta2Z`^ssozD9BINn45NxUeCQis?-BKmU*Kt=FY-NJ+)8S1ecuFtN-M?&42
zl2$G>u!iNhAk*HoJ^4v^9#ORYp5t^wDj6|lx~5w45#E5wVqI1JQ~9l?nPp1YINf++
zMAdSif~_ETv@Er(EFBI^@L4BULFW>)NI+ejHFP*T}UhWNN`I)RRS8za?
z*@`1>9ZB}An%aT5K=_2iQmfE;GcBVHLF!$`I99o5GO`O%O_zLr9AG18>&^HkG(;=V
z%}c!OBQ~?MX(9h~tajX{=x)+!cbM7$YzTlmsPOdp2L-?GoW`@{lY9U3f;OUo*BwRB
z8A+nv(br0-SH#VxGy#ZrgnGD(=@;HME;yd46EgWJ`EL%oXc&lFpc@Y}^>G(W>h_v_
zlN!`idhX+OjL+~T?19sroAFVGfa5tX-D49w$1g2g_-T|EpHL6}K_aX4$K=LTvwtlF
zL*z}j{f+Uoe7{-px3_5iKPA<_7W=>Izkk)!l9ez2w%vi(?Y;i8AxRNLSOGDzNoqoI
zP!1uAl}r=_871(G?y`i&)-7{u=%nxk7CZ_Qh#!|ITec
zwQn`33GTUM`;D2POWnkqngqJhJRlM>CTONzTG}>^Q0wUunQyn|TAiHzyX2_%ATx%P
z%7gW)%4rA9^)M<_%k@`Y?RbC<29sWU&5;@|9thf2#zf8z12$hRcZ!CSb>kUp=4N#y
zl3hE#y6>kkA8VY2`W`g5Ip?2qC_BY$>R`iGQLhz2-S>x(RuWv)SPaGdl^)gGw7tjR
zH@;jwk!jIaCgSg_*9iF|a);sRUTq30(8I(obh^|}S~}P4U^BIGYqcz;MPpC~Y@k_m
zaw4WG1_vz2GdCAX!$_a%GHK**@IrHSkGoN>)e}>yzUTm52on`hYot7cB=oA-h1u|R
ztH$11t?54Qg2L+i33FPFKKRm1aOjKST{l1*(nps`>sv%VqeVMWjl5+Gh+9);hIP8?
zA@$?}Sc
z3qIRpba+y5yf{R6G(u8Z^vkg0Fu&D-7?1s=QZU`Ub{-!Y`I?AGf1VNuc^L3v>)>i#
z{DV9W$)>34wnzAXUiV^ZpYKw>UElrN_5Xj6{r_3|
z$X5PK`e5$7>~9Dj7gK5ash(dvs`vwfk}&RD`>04;j62zoXESkFBklYaKm5seyiX(P
zqQ-;XxlV*yg?Dhlx%xt!b0N3GHp@(p$A;8|%#
zZ5m2KL|{on4nr>2_s9Yh=r5ScQ0;aMF)G$-9-Ca6%wA`Pa)i?NGFA|#Yi?{X-4ZO_
z^}%7%vkzvUHa$-^Y#aA+aiR5sa%S|Ebyn`EV<3Pc?ax_f>@sBZF1S;7y$CXd5t5=WGsTKBk8$OfH4v|0?0I=Yp}7c=WBSCg!{0n)XmiU;lfx)**zZaYqmDJelxk$)nZyx5`x$6R|fz(;u
zEje5Dtm|a%zK!!tk3{i9$I2b{vXNFy%Bf{50X!x{98+BsDr_u9i>G5%*sqEX|06J0
z^IY{UcEbj6LDwuMh7cH`H@9sVt1l1#8kEQ(LyT@&+K}(ReE`ux8gb0r6L_#bDUo^P
z3Ka2lRo52Hdtl_%+pwVs14=q`{d^L58PsU@AMf(hENumaxM{7iAT5sYmWh@hQCO^
zK&}ijo=`VqZ#a3vE?`7QW0ZREL17ZvDfdqKGD?0D4fg{7v%|Yj&_jcKJAB)>=*RS*
zto8p6@k%;&^ZF>hvXm&$PCuEp{uqw3VPG$9VMdW5$w-fy2CNNT>E;>ejBgy-m_6`&
z97L1p{%srn@O_JQgFpa_#f(_)eb#YS>o>q3(*uB;uZb605(iqM$=NK{nHY=+X2*G)
zO3-_Xh%aG}fHWe*==58zBwp%&`mge<8uq8;xIxOd=P%9EK!34^E9sk|(Zq1QSz-JVeP12Fp)-`F|KY$LPwUE?rku
zY@OJ)Z9A!ojfzfeyJ9;zv2EM7ZQB)AR5xGa-tMn^bl)FmoIiVyJ@!~@%{}qXXD&Ns
zPnfe5U+&ohKefILu_1mPfLGuapX@btta5C#gPB2cjk5m4T}Nfi+Vfka!Yd(L?-c~5
z#ZK4VeQEXNPc4r$K00Fg>g#_W!YZ)cJ?JTS<&68_$#cZT-ME`}tcwqg3#``3M3UPvn+pi}(VNNx6y
zFIMVb6OwYU(2`at$gHba*qrMVUl8xk5z-z~fb@Q3Y_+aXuEKH}L+>eW__!IAd@V}L
zkw#s%H0v2k5-=vh$^vPCuAi22Luu3uKTf6fPo?*nvj$9(u)4$6tvF-%IM+3pt*cgs
z_?wW}J7VAA{_~!?))?s6{M=KPpVhg4fNuU*|3THp@_(q!b*hdl{fjRVFWtu^1dV(f
z6iOux9hi&+UK=|%M*~|aqFK{Urfl!TA}UWY#`w(0P!KMe1Si{8|o))Gy6d7;!JQYhgMYmXl?3FfOM2nQGN@~Ap6(G
z3+d_5y@=nkpKAhRqf{qQ~k7Z$v&l&@m7Ppt#FSNzKPZM
z8LhihcE6i=<(#87E|Wr~HKvVWhkll4iSK$^mUHaxgy8*K$_Zj;zJ`L$naPj+^3zTi
z-3NTaaKnD5FPY-~?Tq6QHnmDDRxu0mh0D|zD~Y=vv_qig5r-cIbCpxlju&8Sya)@{
zsmv6XUSi)@(?PvItkiZEeN*)AE~I_?#+Ja-r8$(XiXei2d@Hi7Rx8+rZZb?ZLa{;@*EHeRQ-YDadz~M*YCM4&F-r;E#M+@CSJMJ0oU|PQ^
z=E!HBJDMQ2TN*Y(Ag(ynAL8%^v;=~q?s4plA_hig&5Z0x_^Oab!T)@6kRN$)qEJ6E
zNuQjg|G7iwU(N8pI@_6==0CL;lRh1dQF#wePhmu@hADFd3B5KIH#dx(2A
zp~K&;Xw}F_N6CU~0)QpQk7s$a+LcTOj1%=WXI(U=Dv!6
z{#<#-)2+gCyyv=Jw?Ab#PVkxPDeH|sAxyG`|Ys}A$PW4TdBv%zDz
z^?lwrxWR<%Vzc8Sgt|?FL6ej_*e&rhqJZ3Y>k=X(^dytycR;XDU16}Pc9Vn0>_@H+
zQ;a`GSMEG64=JRAOg%~L)x*w{2re6DVprNp+FcNra4VdNjiaF0M^*>CdPkt(m150rCue?FVdL0nFL$V%5y6N
z%eLr5%YN7D06k5ji5*p4v$UMM)G??Q%RB27IvH7vYr_^3>1D-M66#MN8tWGw>WED}
z5AhlsanO=STFYFs)Il_0i)l)f<8qn|$DW7ZXhf5xI;m+7M5-%P63XFQrG9>DMqHc}
zsgNU9nR`b}E^mL5=@7<1_R~j@q_2U^3h|+`7YH-?C=vme1C3m`Fe0HC>pjt6f_XMh
zy~-i-8R46QNYneL4t@)<0VU7({aUO?aH`z4V2+kxgH5pYD5)wCh75JqQY)jIPN=U6
z+qi8cGiOtXG2tXm;_CfpH9ESCz#i5B(42}rBJJF$jh<1sbpj^8&L;gzGHb8M{of+}
zzF^8VgML2O9nxBW7AvdEt90vp+#kZxWf@A)o9f9}vKJy9NDBjBW
zSt=Hcs=YWCwnfY1UYx*+msp{g!w0HC<_SM!VL1(I2PE?CS}r(eh?{I)mQixmo5^p#
zV?2R!R@3GV6hwTCrfHiK#3Orj>I!GS2kYhk1S;aFBD_}u2v;0HYFq}Iz1Z(I4oca4
zxquja8$+8JW_EagDHf$a1OTk5S97umGSDaj)gH=fLs9>_=XvVj^Xj9a#gLdk=&3tl
zfmK9MNnIX9v{?%xdw7568
zNrZ|roYs(vC4pHB5RJ8>)^*OuyNC>x7ad)tB_}3SgQ96+-JT^Qi<`xi=)_=$Skwv~
zdqeT9Pa`LYvCAn&rMa2aCDV(TMI#PA5g#RtV|CWpgDYRA^|55LLN^uNh*gOU>Z=a06qJ;$C9z8;n-Pq=qZnc1zUwJ@t)L;&NN+E5m
zRkQ(SeM8=l-aoAKGKD>!@?mWTW&~)uF2PYUJ;tB^my`r9n|Ly~0c%diYzqs9W#FTjy?h&X3TnH
zXqA{QI82sdjPO->f=^K^f>N`+B`q9&rN0bOXO79S&a9XX8zund(kW7O76f4dcWhIu
zER`XSMSFbSL>b;Rp#`CuGJ&p$s~G|76){d?xSA5wVg##_O0DrmyEYppyBr%fyWbbv
zp`K84JwRNP$d-pJ!Qk|(RMr?*!wi1if-9G#0p>>1QXKXWFy)eB3ai)l3601q8!9JC
zvU#ZWWDNKq9g6fYs?JQ)Q4C_cgTy3FhgKb8s&m)DdmL5zhNK#8wWg!J*7G7Qhe9VU
zha?^AQTDpYcuN!B+#1dE*X{<#!M%zfUQbj=zLE{dW0XeQ7-oIsGY6RbkP2re@Q{}r_$iiH0xU%iN*ST`A)-EH6eaZB$GA#v)cLi
z*MpA(3bYk$oBDKAzu^kJoSUsDd|856DApz={3u8sbQV@JnRkp2nC|)m;#T=DvIL-O
zI4vh;g7824l}*`_p@MT4+d`JZ2%6NQh=N9bmgJ#q!hK@_<`HQq3}Z8Ij>3%~<*=
zcv=!oT#5xmeGI92lqm9sGVE%#X$ls;St|F#u!?5Y7syhx6q#MVRa&lBmmn%$C0QzU
z);*ldgwwCmzM3uglr}!Z2G+?&
zf%Dpo&mD%2ZcNFiN-Z0f;c_Q;A%f@>26f?{d1kxIJD}LxsQkB47SAdwinfMILZdN3
zfj^HmTzS3Ku5BxY>ANutS8WPQ-G>v4^_Qndy==P3pDm+Xc?>rUHl-4+^%Sp5atOja
z2oP}ftw-rqnb}+khR3CrRg^ibi6?QYk1*i^;kQGirQ=uB9Sd1NTfT-Rbv;hqnY4neE5H1YUrjS2m+2&@uXiAo-
zrKUX|Ohg7(6F(AoP~tj;NZlV#xsfo-5reuQHB$&EIAhyZk;bL;k9ouDmJNBAun;H&
zn;Of1z_Qj`x&M;5X;{s~iGzBQTY^kv-k{ksbE*Dl%Qf%N@hQCfY~iUw!=F-*$cpf2
z3wix|aLBV0b;W@z^%7S{>9Z^T^fLOI68_;l@+Qzaxo`nAI8emTV@rRhEKZ
z?*z_{oGdI~R*#<2{bkz$G~^Qef}$*4OYTgtL$e9q!FY7EqxJ2`zk6SQc}M(k(_MaV
zSLJnTXw&@djco1~a(vhBl^&w=$fa9{Sru>7g8SHahv$&Bl(D@(Zwxo_3r=;VH|uc5
zi1Ny)J!<(KN-EcQ(xlw%PNwK8U>4$9nVOhj(y0l9X^vP1TA>r_7WtSExIOsz`nDOP
zs}d>Vxb2Vo2e5x8p(n~Y5ggAyvib>d)6?)|E@{FIz?G3PVGLf7-;BxaP;c?7ddH$z
zA+{~k^V=bZuXafOv!RPsE1GrR3J2TH9uB=Z67gok+u`V#}BR86hB1xl}H4v`F+mRfr
zYhortD%@IGfh!JB(NUNSDh+qDz?4ztEgCz&bIG-Wg7w-ua4ChgQR_c+z8dT3<1?uX
z*G(DKy_LTl*Ea!%v!RhpCXW1WJO6F`bgS-SB;Xw9#!
z<*K}=#wVu9$`Yo|e!z-CPYH!nj7s9dEPr-E`DXUBu0n!xX~&|%#G=BeM?X@shQQMf
zMvr2!y7p_gD5-!Lnm|a@z8Of^EKboZsTMk%5VsJEm>VsJ4W7Kv{<|#4f-qDE$D-W>gWT%z-!qXnDHhOvLk=?^a1*|0j
z{pW{M0{#1VcR5;F!!fIlLVNh_Gj
zbnW(_j?0c2q$EHIi@fSMR{OUKBcLr{Y&$hrM8XhPByyZaXy|dd&{hYQRJ9@Fn%h3p7*VQolBIV@Eq`=y%5BU~3RPa^$a?ixp^cCg
z+}Q*X+CW9~TL29@OOng(#OAOd!)e$d%sr}^KBJ-?-X&|4HTmtemxmp?cT3uA?md4%
zT8yZ0U;6Rg6JHy3fJae{6TMGS?ZUX6+gGTT{Q{)SI85$5FD{g-eR%O0KMpWPY`4@O
zx!hen1*8^E(*}{m^V_?}(b5k3hYo=T+$&M32+B`}81~KKZhY;2H{7O-M@vbCzuX0n
zW-&HXeyr1%I3$@ns-V1~Lb@wIpkmx|8I~ob1Of7i6BTNysEwI}=!nU%q7(V_^+d*G
z7G;07m(CRTJup!`cdYi93r^+LY+`M*>aMuHJm(A8_O8C#A*$!Xvddgpjx5)?_EB*q
zgE8o5O>e~9IiSC@WtZpF{4Bj2J5eZ>uUzY%TgWF7wdDE!fSQIAWCP)V{;HsU3ap?4
znRsiiDbtN7i9hapO;(|Ew>Ip2TZSvK9Z^N21%J?OiA_&eP1{(Pu_=%JjKy|HOardq
ze?zK^K
zA%sjF64*Wufad%H<)
z^|t>e*h+Z1#l=5wHexzt9HNDNXgM=-OPWKd^5p!~%SIl>Fo&7BvNpbf8{NXmH)o{r
zO=aBJ;meX1^{O%q;kqdw*5k!Y7%t_30
zy{nGRVc&5qt?dBwLs+^Sfp;f`YVMSB#C>z^a9@fpZ!xb|b-JEz1LBX7ci)V@W+kvQ89KWA0T~Lj$aCcfW#nD5bt&Y_<
z-q{4ZXDqVg?|0o)j1%l0^_it0WF*LCn-+)c!2y5yS7aZIN$>0LqNnkujV*YVes(v$
zY@_-!Q;!ZyJ}Bg|G-~w@or&u0RO?vlt5*9~yeoPV_UWrO2J54b4#{D(D>jF(R88u2
zo#B^@iF_%S>{iXSol8jpmsZuJ?+;epg>k=$d`?GSegAVp3n$`GVDvK${N*#L_1`44
z{w0fL{2%)0|E+qgZtjX}itZz^KJt4Y;*8uSK}Ft38+3>j|K(PxIXXR-t4VopXo#9#
zt|F{LWr-?34y`$nLBVV_*UEgA6AUI65dYIbqpNq9cl&uLJ0~L}<=ESlOm?Y-S@L*d
z<7vt}`)TW#f%Rp$Q}6@3=j$7Tze@_uZO@aMn<|si{?S}~maII`VTjs&?}jQ4_cut9$)PEqMukwoXobzaKx^MV
z2fQwl+;LSZ$qy%Tys0oo^K=jOw$!YwCv^ei4NBVauL)tN%=wz9M{uf{IB(BxK|lT*pFkmNK_1tV`nb%jH=a0~VNq2RCKY(rG7jz!-D^k)Ec)yS%17pE#o6&eY+
z^qN(hQT$}5F(=4lgNQhlxj?nB4N6ntUY6(?+R#B?W3hY_a*)hnr4PA|vJ<6p`K3Z5Hy
z{{8(|ux~NLUW=!?9Qe&WXMTAkQnLXg(g=I@(VG3{HE13OaUT|DljyWXPs2FE@?`iU
z4GQlM&Q=T<4&v@Fe<+TuXiZQT3G~vZ&^POfmI1K2h6t4eD}Gk5XFGpbj1n_g*{qmD6Xy
z`6Vv|lLZtLmrnv*{Q%xxtcWVj3K4M%$bdBk_a&ar{{GWyu#ljM;dII;*jP;QH
z#+^o-A4np{@|Mz+LphTD0`FTyxYq#wY)*&Ls5o{0z9yg2K+K7ZN>j1>N&;r+Z`vI|
zDzG1LJZ+sE?m?>x{5LJx^)g&pGEpY=fQ-4}{x=ru;}FL$inHemOg%|R*ZXPodU}Kh
zFEd5#+8rGq$Y<_?k-}r5zgQ3jRV=ooHiF|@z_#D4pKVEmn5CGV(9VKCyG|sT9nc=U
zEoT67R`C->KY8Wp-fEcjjFm^;Cg(ls|*ABVHq8clBE(;~K^b+S>6uj70g?
z&{XQ5U&!Z$SO7zfP+y^8XBbiu*Cv-yJG|l-oe*!s5$@Lh_KpxYL2sx`B|V=dETN>5K+C+CU~a_3cI8{vbu$TNVdGf15*>D
zz@f{zIlorkY>TRh7mKuAlN9A0>N>SV`X)+bEHms=mfYTMWt_AJtz_h+JMmrgH?mZt
zm=lfdF`t^J*XLg7v+iS)XZROygK=CS@CvUaJo&w2W!Wb@aa?~Drtf`JV^cCMjngVZ
zv&xaIBEo8EYWuML+vxCpjjY^s1-ahXJzAV6hTw%ZIy!FjI}aJ+{rE&u#>rs)vzuxz
z+$5z=7W?zH2>Eb32dvgHYZtCAf!=OLY-pb4>Ae79rd68E2LkVPj-|jFeyqtBCCwiW
zkB@kO_(3wFq)7qwV}bA=zD!*@UhT`geq}ITo%@O(Z5Y80nEX~;0-8kO{oB6|(4fQh
z);73T!>3@{ZobPwRv*W?7m0Ml9GmJBCJd&6E?hdj9lV=
z4flNfsc(J*DyPv?RCOx!MSvk(M952PJ-G|JeVxWVjN~SNS6n-_Ge3Q;TGE;EQvZg86%wZ`MB
zSMQua(i*R8a75!6$QRO^(o7sGoomb+Y{OMy;m~Oa`;P9Yqo>?bJAhqXxLr7_3g_n>f#UVtxG!^F#1+y@os6x(sg
z^28bsQ@8rw%Gxk-stAEPRbv^}5sLe=VMbkc@Jjimqjvmd!3E7+QnL>|(^3!R}
zD-l1l7*Amu@j+PWLGHXXaFG0Ct2Q=}5YNUxEQHCAU7gA$sSC<5OGylNnQUa>>l%sM
zyu}z6i&({U@x^hln**o6r2s-(C-L50tQvz|zHTqW!ir?w&V23tuYEDJVV#5pE|OJu
z7^R!A$iM$YCe?8n67l*J-okwfZ+ZTkGvZ)tVPfR;|3gyFjF)8V
zyXXN=!*bpyRg9#~Bg1+UDYCt0
ztp4&?t1X0q>uz;ann$OrZs{5*r`(oNvw=$7O#rD|Wuv*wIi)4b
zGtq4%BX+kkagv3F9Id6~-c+1&?zny%w5j&nk9SQfo0k4LhdSU_kWGW7axkfpgR`8*
z!?UTG*Zi_baA1^0eda8S|@&F
z{)Rad0kiLjB|=}XFJhD(S3ssKlveFFmkN{Vl^_nb!o5M!RC=m)V&v2%e?ZoRC@h3>
zJ(?pvToFd`*Zc@HFPL#=otWKwtuuQ_dT-Hr{S%pQX<6dqVJ8;f(o)4~VM_kEQkMR+
zs1SCVi~k>M`u1u2xc}>#D!V&6nOOh-E$O&SzYrjJdZpaDv1!R-QGA141WjQe2s0J~
zQ;AXG)F+K#K8_5HVqRoRM%^EduqOnS(j2)|ctA6Q^=|s_WJYU;Z%5bHp08HPL`YF2
zR)Ad1z{zh`=sDs^&V}J
z%$Z$!jd7BY5AkT?j`eqMs%!Gm@T8)4w3GYEX~IwgE~`d|@T{WYHkudy(47brgHXx&
zBL1yFG6!!!VOSmDxBpefy2{L_u5yTwja&HA!mYA#wg#bc-m%~8aRR|~AvMnind@zs
zy>wkShe5&*un^zvSOdlVu%kHsEo>@puMQ`b1}(|)l~E{5)f7gC=E$fP(FC2=F<^|A
zxeIm?{EE!3sO!Gr7e{w)Dx(uU#3WrFZ>ibmKSQ1tY?*-Nh1TDHLe+k*;{Rp!Bmd_m
zb#^kh`Y*8l|9Cz2e{;RL%_lg{#^Ar+NH|3z*Zye>!alpt{z;4dFAw^^H!6ING*EFc
z_yqhr8d!;%nHX9AKhFQZBGrSzfzYCi%C!(Q5*~hX>)0N`vbhZ@N|i;_972WSx*>LH
z87?en(;2_`{_JHF`Sv6Wlps;dCcj+8IJ8ca6`DsOQCMb3n#
z3)_w%FuJ3>fjeOOtWyq)ag|PmgQbC-s}KRHG~enBcIwqIiGW8R8jFeBNY9|YswRY5
zjGUxdGgUD26wOpwM#8a!Nuqg68*dG@VM~SbOroL_On0N6QdT9?)NeB3@0FCC?Z|E0
z6TPZj(AsPtwCw>*{eDEE}Gby>0q{*lI+g2e&(YQrsY&uGM{O~}(oM@YWmb*F
zA0^rr5~UD^qmNljq$F#ARXRZ1igP`MQx4aS6*MS;Ot(1L5jF2NJ;de!NujUYg$dr#
z=TEL_zTj2@>ZZN(NYCeVX2==~=aT)R30gETO{G&GM4XN<+!&W&(WcDP%oL8PyIVUC
zs5AvMgh6qr-2?^unB@mXK*Dbil^y-GTC+>&N5HkzXtozVf93m~xOUHn8`HpX=$_v2
z61H;Z1qK9o;>->tb8y%#4H)765W4E>TQ1o0PFj)uTOPEvv&}%(_mG0ISmyhnQV33Z$#&yd{
zc{>8V8XK$3u8}04CmAQ#I@XvtmB*s4t8va?-IY4@CN>;)mLb_4!&P3XSw4pA_NzDb
zORn!blT-aHk1%Jpi>T~oGLuh{DB)JIGZ9KOsciWs2N7mM1JWM+lna4vkDL?Q)z_Ct
z`!mi0jtr+4*L&N7jk&LodVO#6?_qRGVaucqVB8*us6i3BTa^^EI0x%EREQSXV@f!lak6Wf1cNZ8>*artIJ(ADO*=<-an`3zB4d*oO*8D1K!f
z*A@P1bZCNtU=p!742MrAj%&5v%Xp_dSX@4YCw%F|%Dk=u|1BOmo)HsVz)nD5USa
zR~??e61sO(;PR)iaxK{M%QM_rIua9C^4ppVS$qCT9j2%?*em?`4Z;4@>I(c%M&#cH
z>4}*;ej<4cKkbCAjjDsyKS8rIm90O)Jjgyxj5^venBx&7B!xLmzxW3jhj7sR(^3Fz
z84EY|p1NauwXUr;FfZjdaAfh%ivyp+^!jBjJuAaKa!yCq=?T_)R!>16?{~p)FQ3LDoMyG%hL#pR!f@P%*;#90rs_y
z@9}@r1BmM-SJ#DeuqCQk=J?ixDSwL*wh|G#us;dd{H}3*-Y7Tv5m=bQJMcH+_S`zVtf;!0kt*(zwJ
zs+kedTm!A}cMiM!qv(c$o5K%}Yd0|nOd0iLjus&;s0Acvoi-PFrWm?+q9f^FslxGi
z6ywB`QpL$rJzWDg(4)C4+!2cLE}UPCTBLa*_=c#*$b2PWrRN46$y~yST3a2$7hEH=
zNjux+wna^AzQ=KEa_5#9Ph=G1{S0#hh1L3hQ`@HrVnCx{!fw_a0N5xV(iPdKZ-HOM
za)LdgK}1ww*C_>V7hbQnTzjURJL`S%`6nTHcgS+dB6b_;PY1FsrdE8(2K6FN>37!62j_cBlui{jO^$dPkGHV>pXvW0EiOA
zqW`YaSUBWg_v^Y5tPJfWLcLpsA8T
zG)!x>pKMpt!lv3&KV!-um=
zKCir6`bEL_LCFx4Z5bAFXW$g3Cq`?Q%)3q0r852XI*Der*JNuKUZ`C{cCuu8R8nkt
z%pnF>R$uY8L+D!V{s^9>IC+bmt<05h**>49R*#vpM*4i0qRB2uPbg8{{s#9yC;Z18
zD7|4m<9qneQ84uX|J&f-g8a|nFKFt34@Bt{CU`v(SYbbn95Q67*)_Esl_;v291s=9
z+#2F2apZU4Tq=x+?V}CjwD(P=U~d<=mfEFuyPB`Ey82V9G#Sk8H_Ob_RnP3s?)S_3
zr%}Pb?;lt_)Nf>@zX~D~TBr;-LS<1I##8z`;0ZCvI_QbXNh8Iv)$LS=*gHr;}dgb=w5$3k2la1keIm|=7<-JD>)U%=Avl0Vj@+&vxn
zt-)`vJxJr88D&!}2^{GPXc^nmRf#}nb$4MMkBA21GzB`-Or`-3lq^O^svO7Vs~FdM
zv`NvzyG+0T!P8l_&8gH|pzE{N(gv_tgDU7SWeiI-iHC#0Ai%Ixn4&nt{5y3(GQs)i
z&uA;~_0shP$0Wh0VooIeyC|lak__#KVJfxa7*mYmZ22@(<^W}FdKjd*U1CqSjNKW%
z*z$5$=t^+;Ui=MoDW~A7;)Mj%ibX1_p4gu>RC}Z_pl`U*{_z@+HN?AF{_W
z?M_X@o%w8fgFIJ$fIzBeK=v#*`mtY$HC3tqw7q^GCT!P$I%=2N4FY7j9nG8aIm$c9
zeKTxVKN!UJ{#W)zxW|Q^K!3s;(*7Gbn;e@pQBCDS(I|Y0euK#dSQ_W^)sv5pa%<^o
zyu}3d?Lx`)3-n5Sy9r#`I{+t6x%I%G(iewGbvor&I^{lhu-!#}*Q3^itvY(^UWXgvthH52zLy&T+B)Pw;5>4D6>74
zO_EBS)>l!zLTVkX@NDqyN2cXTwsUVao7$HcqV2%t$YzdAC&T)dwzExa3*kt9d(}al
zA~M}=%2NVNUjZiO7c>04YH)sRelXJYpWSn^aC$|Ji|E13a^-v2MB!Nc*b+=KY7MCm
zqIteKfNkONq}uM;PB?vvgQvfKLPMB8u5+Am=d#>g+o&Ysb>dX9EC8q?D$pJH!MTAqa=DS5$cb+;hEvjwVfF{4;M{5U&^_+r
zvZdu_rildI!*|*A$TzJ&apQWV@p{!W`=?t(o0{?9y&vM)V)ycGSlI3`;ps(vf2PUq
zX745#`cmT*ra7XECC0gKkpu2eyhFEUb?;4@X7weEnLjXj_F~?OzL1U1L0|s6M+kIhmi%`n5vvDALMagi4`wMc=JV{XiO+^
z?s9i7;GgrRW{Mx)d7rj)?(;|b-`iBNPqdwtt%32se@?w4<^KU&585_kZ=`Wy^oLu9
z?DQAh5z%q;UkP48jgMFHTf#mj?#z|=w=
z(q6~17Vn}P)J3M?O)x))%a5+>TFW3No~TgP;f}K$#icBh;rSS+R|}l鯊%1Et
zwk~hMkhq;MOw^Q5`7oC{CUUyTw9x>^%*FHx^qJw(LB+E0WBX@{Ghw;)6aA-KyYg8p
z7XDveQOpEr;B4je@2~usI5BlFadedX^ma{b{ypd|RNYqo#~d*mj&y`^iojR}s%~vF
z(H!u`yx68D1Tj(3(m;Q+Ma}s2n#;O~bcB1`lYk%Irx60&-nWIUBr2x&@}@76+*zJ5
ze&4?q8?m%L9c6h=J$WBzbiTf1Z-0Eb5$IZs>lvm$>1n_Mezp*qw_pr8<8$6f)5f<@
zyV#tzMCs51nTv_5ca`x`yfE5YA^*%O_H?;tWYdM_kHPubA%vy47i=9>Bq)
zRQ&0UwLQHeswmB1yP)+BiR;S+Vc-5TX84KUA;8VY9}yEj0eESSO`7HQ4lO
z4(CyA8y1G7_C;6kd4U3K-aNOK!sHE}KL_-^EDl(vB42P$2Km7$WGqNy=%fqB+
zSLdrlcbEH=T@W8V4(TgoXZ*G1_aq$K^@ek=TVhoKRjw;HyI&coln|uRr5mMOy2GXP
zwr*F^Y|!Sjr2YQXX(Fp^*`Wk905K%$bd03R4(igl0&7IIm*#f`A!DCarW9$h$z`kYk9MjjqN&5-DsH@8xh63!fTNPxWsFQhNv
z#|3RjnP$Thdb#Ys7M+v|>AHm0BVTw)EH}>x@_f4zca&3tXJhTZ8pO}aN?(dHo)44Z
z_5j+YP=jMlFqwvf3lq!57-SAuRV2_gJ*wsR_!Y4Z(trO}0wmB9%f#jNDHPdQGHFR;
zZXzS-$`;7DQ5vF~oSgP3bNV$6Z(rwo6W(U07b1n3UHqml>{=6&-4PALATsH@Bh^W?
z)ob%oAPaiw{?9HfMzpGb)@Kys^J$CN{uf*HX?)z=g`J(uK1YO^8~s1(ZIbG%Et(|q
z$D@_QqltVZu9Py4R0Ld8!U|#`5~^M=b>fnHthzKBRr=i+w@0Vr^l|W;=zFT#PJ?*a
zbC}G#It}rQP^Ait^W&aa6B;+0gNvz4cWUMzpv(1gvfw-X4xJ2Sv;mt;zb2Tsn|kSS
zo*U9N?I{=-;a-OybL4r;PolCfiaL=y@o9{%`>+&FI#D^uy#>)R@b^1ue&AKKwuI*`
zx%+6r48EIX6nF4o;>)zhV_8(IEX})NGU6Vs(yslrx{5fII}o3SMHW7wGtK9oIO4OM&@@ECtXSICLcPXoS|{;=_yj>hh*%hP27yZwOmj4&Lh
z*Nd@OMkd!aKReoqNOkp5cW*lC)&C$P?+H3*%8)6HcpBg&IhGP^77XPZpc%WKYLX$T
zsSQ$|ntaVVOoRat$6lvZO(G-QM5s#N4j*|N_;8cc2v_k4n6zx9c1L4JL*83F-C1Cn
zaJhd;>rHXB%%ZN=3_o3&Qd2YOxrK~&?1=UuN9QhL$~OY-Qyg&})#ez*8NpQW_*a&kD&ANjedxT0Ar
z<6r{eaVz3`d~+N~vkMaV8{F?RBVemN(jD@S8qO~L{rUw#=2a$V(7rLE+kGUZ<%pdr
z?$DP|Vg#gZ9S}w((O2NbxzQ^zTot=89!0^~hE{|c9q1hVzv0?YC5s42Yx($;hAp*E
zyoGuRyphQY{Q2ee0Xx`1&lv(l-SeC$NEyS~8iil3_aNlnqF_G|;zt#F%1;J)jnPT&
z@iU0S;wHJ2$f!juqEzPZeZkjcQ+Pa@eERSLKsWf=`{R@yv7AuRh&ALRTAy
z8=g&nxsSJCe!QLchJ=}6|LshnXIK)SNd
zRkJNiqHwKK{SO;N5m5wdL&qK`v|d?5<4!(FAsDxR>Ky#0#t$8XCMptvNo?|SY?d8b
z`*8dVBlXTUanlh6n)!EHf2&PDG8sXNAt6~u-_1EjPI1|<=33T8
zEnA00E!`4Ave0d&VVh0e>)Dc}=FfAFxpsC1u9ATfQ`-Cu;mhc8Z>2;uyXtqpLb7(P
zd2F9<3cXS}
znMg?{&8_YFTGRQZEPU-XPq55%51}RJpw@LO_|)CFAt62-_!u_Uq$csc+7|3+TV_!h
z+2a7Yh^5AA{q^m|=KSJL+w-EWDBc&I_I1vOr^}P8i?cKMhGy$CP0XKrQzCheG$}G#
zuglf8*PAFO8%xop7KSwI8||liTaQ9NCAFarr~psQt)g*pC@9bORZ>m`_GA`_K@~&%
zijH0z;T$fd;-Liw8%EKZas>BH8nYTqsK7F;>>@YsE=Rqo?_8}UO-S#|6~CAW0Oz1}
z3F(1=+#wrBJh4H)9jTQ_$~@#9|Bc1Pd3rAIA_&vOpvvbgDJOM(yNPhJJq2%PCcMaI
zrbe~toYzvkZYQ{ea(Wiyu#4WB#RRN%bMe=SOk!CbJZv^m?Flo5p{W8|0i3`hI3Np#
zvCZqY%o258CI=SGb+A3yJe~JH^i{uU`#U#fvSC~rWTq+K`E%J@
zasU07&pB6A4w3b?d?q}2=0rA#SA7D`X+zg@&zm^iA*HVi
z009#PUH<%lk4z~p^l0S{lCJk1Uxi=F4e_DwlfHA`X`rv(|JqWKAA5nH+u4Da+E_p+
zVmH@lg^n4ixs~*@gm_dgQ&eDmE1mnw5wBz9Yg?QdZwF|an67Xd*x!He)Gc8&2!urh
z4_uXzbYz-aX)X1>&iUjGp;P1u8&7TID0bTH-jCL&Xk8b&;;6p2op_=y^m@Nq*0{#o!!A;wNAFG@0%Z9rHo
zcJs?Th>Ny6+hI`+1XoU*ED$Yf@9f91m9Y=#N(HJP^Y@ZEYR6I?oM{>&Wq4|v0IB(p
zqX#Z<_3X(&{H+{3Tr|sFy}~=bv+l=P;|sBz$wk-n^R`G3p0(p>p=5ahpaD7>r|>pm
zv;V`_IR@tvZreIuv2EM7ZQHhO+qUgw#kOs%*ekY^n|=1#x9&c;Ro&I~{rG-#_3ZB1
z?|9}IFdbP}^DneP*T-JaoYHt~r@EfvnPE5EKUwIxjPbsr$%
zfWW83pgWST7*B(o=kmo)74$8UU)v0{@4DI+ci&%=#90}!CZz|rnH+Mz=HN~97G3~@
z;v5(9_2%eca(9iu@J@aqaMS6*$TMw!S>H(b
z4(*B!|H|8&EuB%mITr~O?vVEf%(Gr)6E=>H~1VR
z&1YOXluJSG1!?TnT)_*YmJ*o_Q@om~(GdrhI{$Fsx_zrkupc#y{DK1WOUR>tk>ZE)
ziOLoBkhZZ?0Uf}cm>GsA>Rd6V8@JF)J*EQlQ<=JD@m<)hyElXR0`pTku*3MU`HJn|
zIf7$)RlK^pW-$87U;431;Ye4Ie+l~_B3*bH1>*yKzn23cH0u(i5pXV!
z4K?{3oF7ZavmmtTq((wtml)m6i)8X6ot_mrE-QJCW}Yn!(3~aUHYG=^fA<^~`e3yc
z-NWTb{gR;DOUcK#zPbN^D*e=2eR^_!(!RKkiwMW@@yYtEoOp4XjOGgzi`;=8
zi3`Ccw1%L*y(FDj=C7Ro-V?q)-%p?Ob2ZElu`eZ99n14-ZkEV#y5C+{Pq87Gu3&>g
zFy~Wk7^6v*)4pF3@F@rE__k3ikx(hzN3@e*^0=KNA6|jC^B5nf(XaoQaZN?Xi}Rn3
z$8&m*KmWvPaUQ(V<#J+S&zO|8P-#!f%7G+n_%sXp9=J%Z4&9OkWXeuZN}ssgQ#Tcj
z8p6ErJQJWZ+fXLCco=RN8D{W%+*kko*2-LEb))xcHwNl~Xmir>kmAxW?eW50Osw3#
zki8Fl$#fvw*7rqd?%E?}ZX4`c5-R&w!Y0#EBbelVXSng+kUfeUiqofPehl}$ormli
zg%r)}?%=?_pHb9`Cq9Z|B`L8b>(!+8HSX?`5+5mm81AFXfnAt1*R3F
z%b2RPIacKAddx%JfQ8l{3U|vK@W7KB$CdLqn@wP^?azRks@x8z59#$Q*7q!KilY-P
zHUbs(IFYRGG1{~@RF;Lqyho$~7^hNC`NL3kn^Td%A7dRgr_&`2k=t+}D-o9&C!y^?
z6MsQ=tc3g0xkK(O%DzR9nbNB(r@L;1zQrs8mzx&4dz}?3KNYozOW5;=w18U6$G4U2
z#2^qRLT*Mo4bV1Oeo1PKQ2WQS2Y-hv&S|C7`xh6=Pj7MNLC5K-zokZ67S)C;(F0Dd
zloDK2_o1$Fmza>EMj3X9je7e%Q`$39Dk~GoOj89-6q9|_WJlSl!!+*{R=tGp
z8u|MuSwm^t7K^nUe+^0G3dkGZr3@(X+TL5eah)K^Tn
zXEtHmR9UIaEYgD5Nhh(s*fcG_lh-mfy5iUF3xxpRZ0q3nZ=1qAtUa?(LnT9I&~uxX
z`pV?+=|-Gl(kz?w!zIieXT}o}7@`QO>;u$Z!QB${a08_bW0_o@&9cjJUXzVyNGCm8
zm=W+$H!;_Kzp6WQqxUI;JlPY&`V}9C$8HZ^m?NvI*JT@~BM=()T()Ii#+*$y@lTZBkmMMda>7s#O(1YZR+zTG@&}!EXFG{
zEWPSDI5bFi;NT>Yj*FjH((=oe%t%xYmE~AGaOc4#9K_XsVpl<4SP@E!TgC0qpe1oi
zNpxU2b0(lEMcoibQ-G^cxO?ySVW26HoBNa;n0}CWL*{k)oBu1>F18X061$SP{Gu67
z-v-Fa=Fl^u3lnGY^o5v)Bux}bNZ~
z5pL+7F_Esoun8^5>z8NFoIdb$sNS&xT8_|`GTe8zSXQzs4r^g0kZjg(b0bJvz`g<70u9Z3fQILX1Lj@;@+##bP|FAOl)U^9U>0rx
zGi)M1(Hce)LAvQO-pW!MN$;#ZMX?VE(22lTlJrk#pB0FJNqVwC+*%${Gt#r_tH9I_
z;+#)#8cWAl?d@R+O+}@1A^hAR1s3UcW{G+>;X4utD2d9X(jF555}!TVN-hByV6t+A
zdFR^aE@GNNgSxxixS2p=on4(+*+f<8xrwAObC)D5)4!z7)}mTpb7&ofF3u&9&wPS<
zB62WHLGMhmrmOAgmJ+|c>qEWTD#jd~lHNgT0?t-p{T=~#EMcB|
z=AoDKOL+qXCfk~F)-Rv**V}}gWFl>liXOl7Uec_8v)(S#av99PX1sQIVZ9eNLkhq$
zt|qu0b?GW_uo}TbU8!jYn8iJeIP)r@;!Ze_7mj{AUV$GEz6bDSDO=D!&C9!M@*S2!
zfGyA|EPlXGMjkH6x7OMF?gKL7{GvGfED=Jte^p=91FpCu)#{whAMw`vSLa`K#atdN
zThnL+7!ZNmP{rc=Z>%$meH;Qi1=m1E3Lq2D_O1-X5C;!I0L>zur@tPAC9*7Jeh)`;eec}1`nkRP(%iv-`N
zZ@ip-g|7l6Hz%j%gcAM}6-nrC8oA$BkOTz^?dakvX?`^=ZkYh%vUE
z9+&)K1UTK=ahYiaNn&G5nHUY5niLGus@p5E2@RwZufRvF{@$hW{;{3QhjvEHMvduO
z#Wf-@oYU4ht?#uP{N3utVzV49mEc9>*TV_W2TVC`6+oI)zAjy$KJrr=*q##&kobiQ
z1vNbya&OVjK`2pdRrM?LuK6BgrLN7H_3m
z!qpNKg~87XgCwb#I=Q&0rI*l$wM!qTkXrx1ko5q-f;=R2fImRMwt5Qs{P*p^z@9ex
z`2#v(qE&F%MXlHpdO#QEZyZftn4f05ab^f2vjxuFaat2}jke{j?5GrF=WYBR?gS(^
z9SBiNi}anzBDBRc+QqizTTQuJrzm^bNA~A{j%ugXP7McZqJ}65l10({wk++$=e8O{
zxWjG!Qp#5OmI#XRQQM?n6?1ztl6^D40hDJr?4$Wc&O_{*OfMfxe)V0=e{|N?J#fgE>j9jAajze$iN!*yeF%jJU#G1c@@rm
zolGW!j?W6Q8pP=lkctNFdfgUMg92wlM4E$aks1??M$~WQfzzzXtS)wKrr2sJeCN4X
zY(X^H_c^PzfcO8Bq(Q*p4c_v@F$Y8cHLrH$`pJ2}=#*8%JYdqsqnGqEdBQMpl!Ot04tUGSXTQdsX&GDtjbWD=prcCT9(+
z&UM%lW%Q3yrl1yiYs;LxzIy>2G}EPY6|sBhL&X&RAQrSAV4Tlh2nITR?{6xO9ujGu
zr*)^E`>o!c=gT*_@6S&>0POxcXYNQd&HMw6<|#{eSute2C3{&h?Ah|cw56-AP^f8l
zT^kvZY$YiH8j)sk7_=;gx)vx-PW`hbSBXJGCTkpt;ap(}G2GY=2bbjABU5)ty%G#x
zAi07{Bjhv}>OD#5zh#$0w;-vvC@^}F!
z#X$@)zIs1L^E;2xDAwEjaXhTBw2<{&JkF*`;c3<1U@A4MaLPe{M5DGGkL}#{cHL%*
zYMG+-Fm0#qzPL#V)TvQVI|?_M>=zVJr9>(6ib*#z8q@mYKXDP`k&A4A};xMK0h=yrMp~JW{L?mE~ph&1Y1a#4%SO)@{
zK2juwynUOC)U*hVlJU17%llUxAJFuKZh3K0gU`aP)pc~bE~mM!i1mi!~LTf>1Wp<
zuG+ahp^gH8g8-M$u{HUWh0m^9Rg@cQ{&DAO{PTMudV6c?ka7+AO&
z746QylZ&Oj`1aqfu?l&zGtJnpEQOt;OAFq19MXTcI~`ZcoZmyMrIKDFRIDi`FH)w;
z8+*8tdevMDv*VtQi|e}CnB_JWs>fhLOH-+Os2Lh!&)Oh2utl{*AwR)QVLS49iTp{6
z;|172Jl!Ml17unF+pd+Ff@jIE-{Oxv)5|pOm@CkHW?{l}b@1>Pe!l}VccX#xp@xgJ
zyE<&ep$=*vT=}7vtvif0B?9xw_3Gej7mN*dOHdQPtW5kA5_zGD
zpA4tV2*0E^OUimSsV#?Tg#oiQ>%4D@1F5@AHwT8Kgen$bSMHD3sXCkq8^(uo7CWk`mT
zuslYq`6Yz;L%wJh$3l1%SZv#QnG3=NZ=BK4yzk#HAPbqXa92;3K5?0kn4TQ`%E%X}
z&>Lbt!!QclYKd6+J7Nl@xv!uD%)*bY-;p`y^ZCC<%LEHUi$l5biu!sT3TGGSTPA21
zT8@B&a0lJHVn1I$I3I1I{W9fJAYc+8
zVj8>HvD}&O`TqU2AAb={?eT;0hyL(R{|h23=4fDSZKC32;wWxsVj`P
z3J3{M$PwdH!ro*Cn!D&=jnFR>BNGR<<|I8CI@+@658Dy(lhqbhXfPTVecY@L8%`3Q
z1Fux2w?2C3th60jI~%OC9BtpNF$QPqcG+Pz96qZJ71_`0o0w_q7|h&O>`6U+^BA&5
zXd5Zp1Xkw~>M%RixTm&OqpNl8Q+ue=92Op_>T~_9UON?ZM2c0aGm=^A4ejrXj3dV9
zhh_bCt-b9`uOX#cFLj!vhZ#lS8Tc47OH>*)y#{O9?AT~KR9LntM|#l#Dlm^8{nZdk
zjMl#>ZM%#^nK2TPzLcKxqx24P7R1FPlBy7LSBrRvx>fE$9AJ;7{PQm~^LBX^k#6Zq
zw*Z(zJC|`!6_)EFR}8|n8&&Rbj8y028~P~sFXBFRt+tmqH-S3<%N;C&WGH!f3{7cm
zy_fCAb9@HqaXa1Y5vFbxWf%#zg6SI$C+Uz5=CTO}e|2fjWkZ;Dx|84Ow~bkI=LW+U
zuq;KSv9VMboRvs9)}2PAO|b(JCEC_A0wq{uEj|3x@}*=bOd
zwr{TgeCGG>HT<@Zeq8y}vTpwDg#UBvD)BEs@1KP$^3$sh&_joQPn{hjBXmLPJ{tC)
z*HS`*2+VtJO{|e$mM^|qv1R*8i(m1`%)}g=SU#T#0KlTM2RSvYUc1fP+va|4;5}Bfz98UvDCpq7}+SMV&;nX
zQw~N6qOX{P55{#LQkrZk(e5YGzr|(B;Q;ju;2a`q+S9bsEH@i1{_Y0;hWYn1-79jl
z5c&bytD*k)GqrVcHn6t-7kinadiD>B{Tl`ZY@`g|b~pvHh5!gKP4({rp?D0aFd_cN
zhHRo4dd5^S6ViN(>(28qZT6E>??aRhc($kP`>@<+lIKS5HdhjVU;>f7<4))E*5|g{
z&d1}D|vpuV^eRj5j|xx9nwaCxXFG?Qbjn~_WSy=N}P0W>MP
zG-F%70lX5Xr$a)2i6?i|iMyM|;Jtf*hO?=Jxj12oz&>P=1#h~lf%#fc73M2_(SUM-
zf&qnjS80|_Y0lDgl&I?*eMumUklLe_=Td!9G@eR*tcPOgIShJipp3{A10u(4eT~DY
zHezEj8V+7m!knn7)W!-5QI3=IvC^as5+TW1@Ern@yX|
z7Nn~xVx&fGSr+L%4iohtS3w^{-H1A_5=r&x8}R!YZvp<2T^YFvj8G_vm}5q;^UOJf
ztl=X3iL;;^^a#`t{Ae-%5Oq{?M#s6Npj+L(n-*LMI-yMR{)qki!~{5z{&`-iL}lgW
zxo+tnvICK=lImjV$Z|O_cYj_PlEYCzu-XBz&XC-JVxUh9;6*z4fuBG+H{voCC;`~GYV|hj%j_&I
zDZCj>Q_0RCwFauYoVMiUSB+*Mx`tg)bWmM^SwMA+?lBg12QUF_x2b)b?qb88K-YUd
z0dO}3k#QirBV<5%jL$#wlf!60dizu;tsp(7XLdI=eQs?P`tOZYMjVq&jE)qK*6B^$
zBe>VvH5TO>s>izhwJJ$<`a8fakTL!yM^Zfr2hV9`f}}VVUXK39p@G|xYRz{fTI+Yq
z20d=)iwjuG9RB$%$^&8#(c0_j0t_C~^|n+c`Apu|x7~;#cS-s=X1|C*YxX3ailhg_|0`g!E&GZJEr?bh#Tpb8siR=JxWKc{#w7g
zWznLwi;zLFmM1g8V5-P#RsM@iX>TK$xsWuujcsVR^7TQ@!+vCD<>Bk9tdCo7Mzgq5
zv8d>dK9x8C@Qoh01u@3h0X_`SZluTb@5o;{4{{eF!-4405x8X7hewZWpz
z2qEi4UTiXTvsa(0X7kQH{3VMF>W|6;6iTrrYD2fMggFA&-CBEfSqPlQDxqsa>{e2M
z(R5PJ7uOooFc|9GU0ELA%m4&4Ja#cQpNw8i8ACAoK6?-px+oBl_yKmenZut#Xumjz
zk8p^OV2KY&?5MUwGrBOo?ki`Sxo#?-Q4gw*Sh0k`@
zFTaYK2;}%Zk-68`#5DXU$2#=%YL#S&MTN8bF+!J2VT6x^XBci6O)Q#JfW{YMz)
zOBM>t2rSj)n#0a3cjvu}r|k3od6W(SN}V-cL?bi*Iz-8uOcCcsX0L>ZXjLqk
zZu2uHq5B|Kt>e+=pPKu=1P@1r9WLgYFq_TNV1p9pu0erHGd!+bBp!qGi+~4A(RsYN@CyXNrC&hxGmW)u5m35OmWwX`I+0yByglO`}HC4nGE^_HUs^&A(uaM
zKPj^=qI{&ayOq#z=p&pnx@@k&I1JI>cttJcu@Ihljt?6p^6{|ds`0MoQwp+I{3l6`
zB<9S((RpLG^>=Kic`1LnhpW2=Gu!x`m~=y;A`Qk!-w`IN;S8S930#vBVMv2vCKi}u
z6<-VPrU0AnE&vzwV(CFC0gnZYcpa-l5T0ZS$P6(?9AM;`Aj~XDvt;Jua=jIgF=Fm?
zdp=M$>`phx%+Gu};;-&7T|B1AcC#L4@mW5SV_^1BRbo6;2PWe$r+npRV`yc;T1mo&
z+~_?7rA+(Um&o@Tddl
zL_hxvWk~a)yY}%j`Y+200D%9$bWHy&;(yj{jpi?Rtz{J66ANw)UyPOm;t6FzY3$hx
zcn)Ir79nhFvNa7^a{SHN7XH*|Vlsx`CddPnA&Qvh8aNhEA;mPVv;Ah=k<*u!Zq^7
z<=xs*iQTQOMMcg|(NA_auh@x`3#_LFt=)}%SQppP{E>mu_LgquAWvh<>L7tf9+~rO
znwUDS52u)OtY<~!d$;m9+87aO+&`#2ICl@Y>&F{jI=H(K+@3M1$rr=*H^dye#~TyD
z!){#Pyfn+|ugUu}G;a~!&&0aqQ59U@UT3|_JuBlYUpT$2+11;}JBJ`{+lQN9T@QFY
z5+`t;6(TS0F?OlBTE!@7D`8#URDNqx2t6`GZ{ZgXeS@v%-eJzZOHz18aS|svxII$a
zZeFjrJ*$IwX$f-Rzr_G>xbu@euGl)B7pC&S+CmDJBg$BoV~jxSO#>y
z33`bupN#LDoW0feZe0%q8un0rYN|eRAnwDHQ6e_)xBTbtoZtTA=Fvk){q}9Os~6mQ
zKB80VI_&6iSq`LnK7*kfHZoeX6?WE}8yjuDn=2#JG$+;-TOA1%^=DnXx%w{b=w}tS
zQbU3XxtOI8E(!%`64r2`zog;5<0b4i)xBmGP^jiDZ2%HNSxIf3@wKs~uk4%3Mxz;~
zts_S~E4>W+YwI<-*-$U8*^HKDEa8oLbmqGg?3vewnaNg%Mm)W=)lcC_J+1ov^u*N3
zXJ?!BrH-+wGYziJq2Y#vyry6Z>NPgkEk+Ke`^DvNRdb>Q2Nlr#v%O@<5hbflI6EKE
z9dWc0-ORk^T}jP!nkJ1imyjdVX@GrjOs%cpgA8-c&FH&$(4od#x6Y&=LiJZPINVyW
z0snY$8JW@>tc2}DlrD3StQmA0Twck~@>8dSix9CyQOALcREdxoM$Sw*l!}bXKq9&r
zysMWR@%OY24@e`?+#xV2bk{T^C_xSo8v2ZI=lBI*l{RciPwuE>L5@uhz@{!l)rtVlWC>)6(G)1~n=Q|S!{E9~6*fdpa*n
z!()-8EpTdj=zr_Lswi;#{TxbtH$8*G=UM`I+icz7sr_SdnHXrv=?iEOF1UL+*6O;%
zPw>t^kbW9X@oEXx<97%lBm-9?O_7L!DeD)Me#rwE54t~UBu9VZ
zl_I1tBB~>jm@bw0Aljz8!
zXBB6ATG6iByKIxs!qr%pz%wgqbg(l{65DP4#v(vqhhL{0b#0C8mq`bnqZ1OwFV
z7mlZZJFMACm>h9v^2J9+^_zc1=JjL#qM5ZHaThH&n
zXPTsR8(+)cj&>Un{6v*z?@VTLr{TmZ@-fY%*o2G}*G}#!bmqpoo*Ay@U!JI^Q@7gj;Kg-HIrLj4}#ec4~D2~X6vo;ghep-@&yOivYP
zC19L0D`jjKy1Yi-SGPAn94(768Tcf$urAf{)1)9W58P`6MA{YG%O?|07!g9(b`8PXG1B1Sh0?HQmeJtP0M$O$hI
z{5G`&9XzYhh|y@qsF1GnHN|~^ru~HVf#)lOTSrv=S@DyR$UKQk
zjdEPFDz{uHM&UM;=mG!xKvp;xAGHOBo~>_=WFTmh$chpC7c`~7?36h)7$fF~Ii}8q
zF|YXxH-Z?d+Q+27Rs3X9S&K3N+)OBxMHn1u(vlrUC6ckBY@@jl+mgr#KQUKo#VeFm
zFwNYgv0<%~Wn}KeLeD9e1$S>jhOq&(e*I@L<=I5b(?G(zpqI*WBqf|Zge0&aoDUsC
zngMRA_Kt0>La+Erl=Uv_J^p(z=!?XHpenzn$%EA`JIq#yYF?JLDMYiPfM(&Csr#f{
zdd+LJL1by?xz|D8+(fgzRs~(N1k9DSyK@LJygwaYX8dZl0W!I&c^K?7)z{2is;OkE
zd$VK-(uH#AUaZrp=1z;O*n=b?QJkxu`Xsw&7yrX0?(CX=I-C#T;yi8a<{E~?vr3W>
zQrpPqOW2M+AnZ&p{hqmHZU-;Q(7?-
zP8L|Q0RM~sB0w1w53f&Kd*y}ofx@c
z5Y6B8qGel+uT1JMot$nT1!Tim6{>oZzJXdyA+4euOLME?5Fd_85Uk%#E*ln%y{u8Q
z$|?|R@Hpb~yTVK-Yr_S#%NUy7EBfYGAg>b({J|5b+j-PBpPy$Ns`PaJin4JdRfOaS
zE|<HjH%NuJgsd2wOlv>~y=np%=2)$M9LS|>P)zJ+Fei5vYo_N~B0XCn+GM76
z)Xz3tg*FRVFgIl9zpESgdpWAavvVViGlU8|UFY{{gVJskg*I!ZjWyk~OW-Td4(mZ6
zB&SQreAAMqwp}rjy`HsG({l2&q5Y52<@AULVAu~rWI$UbFuZs>Sc*x+XI<+ez%$U)|a^unjpiW0l0
zj1!K0(b6$8LOjzRqQ~K&dfbMIE=TF}XFAi)$+h}5SD3lo
z%%Qd>p9se=VtQG{kQ;N`sI)G^u|DN#7{aoEd
zkksYP%_X$Rq08);-s6o>CGJ<}v`qs%eYf+J%DQ^2k68C%nvikRsN?$ap--f+vCS`K
z#&~)f7!N^;sdUXu54gl3L=LN>FB^tuK=y2e#|hWiWUls__n@L|>xH{%8lIJTd5`w?
zSwZbnS;W~DawT4OwSJVdAylbY+u5S+ZH{4hAi2&}Iv~W(UvHg(1GTZRPz`@{SOqzy
z(8g&Dz=$PfRV=6FgxN~zo+G8OoPI&d-thcGVR*_^(R8COTM@bq?fDwY{}WhsQS1AK
zF6R1t8!RdFmfocpJ6?9Yv~;WYi~XPgs(|>{5})j!AR!voO7y9&cMPo#80A(`za@t>cx<0;qxM@S*m(jYP)dMXr*?q0E`oL;12}VAep179uEr8c<=D
zr5?A*C{eJ`z9Ee;E$8)MECqatHkbHH
z&Y+ho0B$31MIB-xm&;xyaFCtg<{m~M-QDbY)fQ>Q*Xibb~8ytxZQ?QMf9!%cV
zU0_X1@b4d+Pg#R!`OJ~DOrQz3@cpiGy~XSKjZQQ|^4J1puvwKeScrH8o{bscBsowomu
z^f12kTvje`yEI3eEXDHJ6L+O{Jv$HVj%IKb|J{IvD*l6IG8WUgDJ*UGz
z3!C%>?=dlfSJ>4U88)V+`U-!9r^@AxJBx8R;)J4Fn@`~k>8>v0M9xp90OJElWP&R5
zM#v*vtT}*Gm1^)Bv!s72T3PB0yVIjJW)H7a)ilkAvoaH?)jjb`MP>2z{%Y?}83
zUIwBKn`-MSg)=?R)1Q0z3b>dHE^)D8LFs}6ASG1|daDly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c
zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d
zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U
zIglK}sn^)W7aB(Q>HvrX=rxB
z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J
z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j
zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5
z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k
zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>&
z9kV9jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM
zTzh^cc+=VNZ&(OdN=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A;
zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT
zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3MEby
zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw
zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7VI5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c
zm!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9
zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM
zeZcOYpm5>U2fU7V*h;%n`8
zN95QhfD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65
zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe
z%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a`
zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0I#KP48tiAPYY!TdW(o|KtVI|EUB9V`CBBNaBLVih7+yMVF|GSoIQD0Jfb{
z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX
zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQz!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm
zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr
z0My%7RFh6ZLiZ_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE
z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e
zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO
zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy?
z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO
z3y0BouW$N&TnwU6!%zS%nIrnANvZF&vB1~P5_d`x-giHuG
zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO
zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*##
z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E
z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^c}
zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl
zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e
zELMFcZtJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1
z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU
zS;a9m^DFMXR4?*X=}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB
z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC
z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X
zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d
zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e6m_ozRF&zux2mlK=v_(_s^R6b5lu?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U
zs0PO&*h_
zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2EF~Th#eb_TueZC`
zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H
z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU?
z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y
z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm
z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n
zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS
ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u
zyv%Yo9~yxQJWJSfj
z?#HY$y=O~F|2pZs22pu|_&Ajd+D(Mt!nPUG{|1nlvP`=R#kKH
zO*s$r_%ss5h1YO7k0bHJ2CXN)Yd6CHn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he
z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3
zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8
z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@
z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n>
z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|=
z&EeIIkw*+$Xd!)j8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq
zM`!pjf(Lm(`$e3FLOjqA5LnN5o!}z{
zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#;
z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc0k4<>0+ClX7%0
zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX
zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z
zR@Pq55mbCQbz+Mn$d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H
zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb
zRwn{Qo@0`7aPa2LQOP}j9v>sjOo5Kqvn|`FLizX
zB+@-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2
zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y;
z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z
zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT
z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v
z;wpDCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s
z8zI4XFQys}q)<`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R
z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6
z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV
zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf
zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__
zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuXrHZxXsI~O
zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}>
zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0#
z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS
zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^YcPpIz8QYM-#s+n%
z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d
zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89
zs=ld@|#;dq1*vQGz=7--Br-|l)
zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h
zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc
zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C
z*$$2b4t%a(e+%>a>d_f_<JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD|
zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU<
zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H
zJvRD^+ps46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=+xzw55&2?darw=9g~>P
z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*qjCVa?OIBj!fW
zm|g?>CVfGXNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f
zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy
z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp
zP9XTHq5mmS=iWu~9ES>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm
zHRVR98*{C?_XNpJn{abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED
zCu^l7u3E%STI}c4{J2!)9SUlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy
z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx
zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3
zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq
z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY
znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5QD7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF
zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0Wq
zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢
zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8fuX!I}#8g+(wxzQwUT#Xb2(tbY1+EUhGKoT@KEU9Ktl>_0
z%bjDJg;#*gtJZv!-Zs`?^}v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+
z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6
z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e
zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA
z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f6U@I2
zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC
zi<%2x8Q24YN+&8U@CIlN
zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8
zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu
z5D)~HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f
z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN
z@GXF$bL=&pBq-#vkTkh>7hl%K5|3
z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3|
zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF
z*L1+}JU7%QJOg|5|Tc|D8fN
zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A
zU*i3&Vg?TluO@X0O;r2Jl6LKLUOVhSqg1*qOt^|8*c7
zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;<
je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc

literal 0
HcmV?d00001

diff --git a/scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.properties b/scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 000000000000..d4081da476bb
--- /dev/null
+++ b/scratch/iceberg-scale-test/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/scratch/iceberg-scale-test/gradlew b/scratch/iceberg-scale-test/gradlew
new file mode 100755
index 000000000000..1aa94a426907
--- /dev/null
+++ b/scratch/iceberg-scale-test/gradlew
@@ -0,0 +1,249 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+#
+#   Gradle start up script for POSIX generated by Gradle.
+#
+#   Important for running:
+#
+#   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+#       noncompliant, but you have some other compliant shell such as ksh or
+#       bash, then to run this script, type that shell name before the whole
+#       command line, like:
+#
+#           ksh Gradle
+#
+#       Busybox and similar reduced shells will NOT work, because this script
+#       requires all of these POSIX shell features:
+#         * functions;
+#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+#         * compound commands having a testable exit status, especially «case»;
+#         * various built-in commands including «command», «set», and «ulimit».
+#
+#   Important for patching:
+#
+#   (2) This script targets any POSIX shell, so it avoids extensions provided
+#       by Bash, Ksh, etc; in particular arrays are avoided.
+#
+#       The "traditional" practice of packing multiple parameters into a
+#       space-separated string is a well documented source of bugs and security
+#       problems, so this is (mostly) avoided, by progressively accumulating
+#       options in "$@", and eventually passing that to Java.
+#
+#       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+#       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+#       see the in-line comments for details.
+#
+#       There are tweaks for specific operating systems such as AIX, CygWin,
+#       Darwin, MinGW, and NonStop.
+#
+#   (3) This script is generated from the Groovy template
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       within the Gradle project.
+#
+#       You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+    APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
+    [ -h "$app_path" ]
+do
+    ls=$( ls -ld "$app_path" )
+    link=${ls#*' -> '}
+    case $link in             #(
+      /*)   app_path=$link ;; #(
+      *)    app_path=$APP_HOME$link ;;
+    esac
+done
+
+# This is normally unused
+# shellcheck disable=SC2034
+APP_BASE_NAME=${0##*/}
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+    echo "$*"
+} >&2
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in                #(
+  CYGWIN* )         cygwin=true  ;; #(
+  Darwin* )         darwin=true  ;; #(
+  MSYS* | MINGW* )  msys=true    ;; #(
+  NONSTOP* )        nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD=$JAVA_HOME/jre/sh/java
+    else
+        JAVACMD=$JAVA_HOME/bin/java
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD=java
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+    case $MAX_FD in #(
+      max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
+        MAX_FD=$( ulimit -H -n ) ||
+            warn "Could not query maximum file descriptor limit"
+    esac
+    case $MAX_FD in  #(
+      '' | soft) :;; #(
+      *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
+        ulimit -n "$MAX_FD" ||
+            warn "Could not set maximum file descriptor limit to $MAX_FD"
+    esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+#   * args from the command line
+#   * the main class name
+#   * -classpath
+#   * -D...appname settings
+#   * --module-path (only if needed)
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+    APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+    JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    for arg do
+        if
+            case $arg in                                #(
+              -*)   false ;;                            # don't mess with options #(
+              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
+                    [ -e "$t" ] ;;                      #(
+              *)    false ;;
+            esac
+        then
+            arg=$( cygpath --path --ignore --mixed "$arg" )
+        fi
+        # Roll the args list around exactly as many times as the number of
+        # args, so each arg winds up back in the position where it started, but
+        # possibly modified.
+        #
+        # NB: a `for` loop captures its iteration list before it begins, so
+        # changing the positional parameters here affects neither the number of
+        # iterations, nor the values presented in `arg`.
+        shift                   # remove old arg
+        set -- "$@" "$arg"      # push replacement arg
+    done
+fi
+
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
+
+set -- \
+        "-Dorg.gradle.appname=$APP_BASE_NAME" \
+        -classpath "$CLASSPATH" \
+        org.gradle.wrapper.GradleWrapperMain \
+        "$@"
+
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+#   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+#   set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+        printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+        xargs -n1 |
+        sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+        tr '\n' ' '
+    )" '"$@"'
+
+exec "$JAVACMD" "$@"
diff --git a/scratch/iceberg-scale-test/settings.gradle b/scratch/iceberg-scale-test/settings.gradle
new file mode 100644
index 000000000000..fdd018dc9055
--- /dev/null
+++ b/scratch/iceberg-scale-test/settings.gradle
@@ -0,0 +1 @@
+rootProject.name = 'iceberg-scale-test'
diff --git a/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java
new file mode 100644
index 000000000000..4cea9312f6fb
--- /dev/null
+++ b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.  You may obtain a copy of the
+ * License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg;
+
+import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.RowCoder;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.iceberg.DistributionMode;
+import org.apache.iceberg.PartitionKey;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.Instant;
+
+/**
+ * Assigns destination metadata for each input record.
+ *
+ * 
The output will have the format { {destination, partition}, data }
+ */
+class AssignDestinationsAndPartitions
+    extends PTransform, PCollection>> {
+
+  private final DynamicDestinations dynamicDestinations;
+  private final IcebergCatalogConfig catalogConfig;
+  private final DistributionMode distributionMode;
+  private final @Nullable SerializableFunction distributionFunction;
+
+  static final String DESTINATION = "destination";
+  static final String PARTITION = "partition";
+  static final String SHARD = "shard";
+  static final org.apache.beam.sdk.schemas.Schema OUTPUT_SCHEMA =
+      org.apache.beam.sdk.schemas.Schema.builder()
+          .addStringField(DESTINATION)
+          .addStringField(PARTITION)
+          .addNullableField(SHARD, org.apache.beam.sdk.schemas.Schema.FieldType.INT32)
+          .build();
+
+  public AssignDestinationsAndPartitions(
+      DynamicDestinations dynamicDestinations, IcebergCatalogConfig catalogConfig) {
+    this(dynamicDestinations, catalogConfig, DistributionMode.HASH, null);
+  }
+
+  public AssignDestinationsAndPartitions(
+      DynamicDestinations dynamicDestinations,
+      IcebergCatalogConfig catalogConfig,
+      DistributionMode distributionMode,
+      @Nullable SerializableFunction distributionFunction) {
+    this.dynamicDestinations = dynamicDestinations;
+    this.catalogConfig = catalogConfig;
+    this.distributionMode = distributionMode;
+    this.distributionFunction = distributionFunction;
+  }
+
+  @Override
+  public PCollection> expand(PCollection input) {
+    return input
+        .apply(
+            ParDo.of(
+                new AssignDoFn(
+                    dynamicDestinations, catalogConfig, distributionMode, distributionFunction)))
+        .setCoder(
+            KvCoder.of(
+                RowCoder.of(OUTPUT_SCHEMA), RowCoder.of(dynamicDestinations.getDataSchema())));
+  }
+
+  @SuppressWarnings("nullness")
+  static class AssignDoFn extends DoFn> {
+    private transient @MonotonicNonNull Map partitionKeys;
+    private transient @MonotonicNonNull Map wrappers;
+    private final DynamicDestinations dynamicDestinations;
+    private final IcebergCatalogConfig catalogConfig;
+    private final DistributionMode distributionMode;
+    private final @Nullable SerializableFunction distributionFunction;
+
+    AssignDoFn(
+        DynamicDestinations dynamicDestinations,
+        IcebergCatalogConfig catalogConfig,
+        DistributionMode distributionMode,
+        @Nullable SerializableFunction distributionFunction) {
+      this.dynamicDestinations = dynamicDestinations;
+      this.catalogConfig = catalogConfig;
+      this.distributionMode = distributionMode;
+      this.distributionFunction = distributionFunction;
+    }
+
+    @Setup
+    public void setup() {
+      this.wrappers = new HashMap<>();
+      this.partitionKeys = new HashMap<>();
+    }
+
+    @ProcessElement
+    public void processElement(
+        @Element Row element,
+        BoundedWindow window,
+        PaneInfo paneInfo,
+        @Timestamp Instant timestamp,
+        OutputReceiver> out) {
+      String tableIdentifier =
+          dynamicDestinations.getTableStringIdentifier(
+              ValueInSingleWindow.of(element, timestamp, window, paneInfo));
+      Row data = dynamicDestinations.getData(element);
+
+      @Nullable PartitionKey partitionKey = checkStateNotNull(partitionKeys).get(tableIdentifier);
+      @Nullable BeamRowWrapper wrapper = checkStateNotNull(wrappers).get(tableIdentifier);
+      if (partitionKey == null || wrapper == null) {
+        PartitionSpec spec = PartitionSpec.unpartitioned();
+        Schema schema = IcebergUtils.beamSchemaToIcebergSchema(data.getSchema());
+        @Nullable
+        IcebergTableCreateConfig createConfig =
+            dynamicDestinations.instantiateDestination(tableIdentifier).getTableCreateConfig();
+        if (createConfig != null && createConfig.getPartitionFields() != null) {
+          spec =
+              PartitionUtils.toPartitionSpec(createConfig.getPartitionFields(), data.getSchema());
+        } else {
+          try {
+            // see if table already exists with a spec
+            // TODO(https://github.com/apache/beam/issues/38337): improve this by periodically
+            // refreshing the table to fetch updated specs
+            spec = catalogConfig.catalog().loadTable(TableIdentifier.parse(tableIdentifier)).spec();
+          } catch (NoSuchTableException ignored) {
+            // no partition to apply
+          }
+        }
+        partitionKey = new PartitionKey(spec, schema);
+        wrapper = new BeamRowWrapper(data.getSchema(), schema.asStruct());
+        checkStateNotNull(partitionKeys).put(tableIdentifier, partitionKey);
+        checkStateNotNull(wrappers).put(tableIdentifier, wrapper);
+      }
+      partitionKey.partition(wrapper.wrap(data));
+      String partitionPath = partitionKey.toPath();
+
+      Integer shardId = null;
+      if (distributionMode == DistributionMode.RANGE && distributionFunction != null) {
+        shardId = distributionFunction.apply(data);
+      }
+
+      Row destAndPartition =
+          Row.withSchema(OUTPUT_SCHEMA)
+              .withFieldValue(DESTINATION, tableIdentifier)
+              .withFieldValue(PARTITION, partitionPath)
+              .withFieldValue(SHARD, shardId)
+              .build();
+      out.output(KV.of(destAndPartition, data));
+    }
+  }
+}
diff --git a/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
new file mode 100644
index 000000000000..4661c1aca209
--- /dev/null
+++ b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
@@ -0,0 +1,887 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg;
+
+import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
+
+import com.google.auto.value.AutoValue;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates;
+import org.apache.iceberg.DistributionMode;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.Duration;
+
+/**
+ * A connector that reads and writes to Apache Iceberg
+ * tables.
+ *
+ * 
{@link IcebergIO} is offered as a Managed transform. This class is subject to change and
+ * should not be used directly. Instead, use it like so:
+ *
+ * 
{@code
+ * Map config = Map.of(
+ *         "table", table,
+ *         "catalog_name", name,
+ *         "catalog_properties", Map.of(
+ *                 "warehouse", warehouse_path,
+ *                 "catalog-impl", "org.apache.iceberg.hive.HiveCatalog"),
+ *         "config_properties", Map.of(
+ *                 "hive.metastore.uris", metastore_uri));
+ *
+ *
+ * ====== WRITE ======
+ * pipeline
+ *     .apply(Create.of(BEAM_ROWS))
+ *     .apply(Managed.write(ICEBERG).withConfig(config));
+ *
+ *
+ * ====== READ ======
+ * pipeline
+ *     .apply(Managed.read(ICEBERG).withConfig(config))
+ *     .getSinglePCollection()
+ *     .apply(ParDo.of(...));
+ *
+ *
+ * ====== READ CDC ======
+ * pipeline
+ *     .apply(Managed.read(ICEBERG_CDC).withConfig(config))
+ *     .getSinglePCollection()
+ *     .apply(ParDo.of(...));
+ * }

+ *
+ * Look for more detailed examples below.
+ *
+ * 
Configuration Options

+ *
+ * Please check the Managed IO
+ * configuration page
+ *
+ * 
Beam Rows

+ *
+ * 
Being a Managed transform, this IO exclusively writes and reads using Beam {@link Row}s.
+ * Conversion takes place between Beam {@link Row}s and Iceberg {@link Record}s using helper methods
+ * in {@link IcebergUtils}. Below is the mapping between Beam and Iceberg types:
+ *
+ * 
+ *   
+ *     
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ * 
 Beam {@link Schema.FieldType}   Iceberg {@link Type}
+ *   
 BYTES   BINARY 
 BOOLEAN   BOOLEAN 
 STRING   STRING 
 INT32   INTEGER 
 INT64   LONG 
 DECIMAL   STRING 
 FLOAT   FLOAT 
 DOUBLE   DOUBLE 
 SqlTypes.DATETIME   TIMESTAMP 
 DATETIME   TIMESTAMPTZ 
 SqlTypes.DATE   DATE 
 SqlTypes.TIME   TIME 
 ITERABLE   LIST 
 ARRAY   LIST 
 MAP   MAP 
 ROW   STRUCT 

+ *
+ * 
Note: {@code SqlTypes} are Beam logical types.
+ *
+ * 
Note on timestamps

+ *
+ * 
For an existing table, the following Beam types are supported for both {@code timestamp} and
+ * {@code timestamptz}:
+ *
+ * 
    
+ *   
  • {@code SqlTypes.DATETIME} --> Using a {@link java.time.LocalDateTime} object
+ *   
  • {@code DATETIME} --> Using a {@link org.joda.time.DateTime} object
+ *   
  • {@code INT64} --> Using a {@link Long} representing micros since EPOCH
+ *   
  • {@code STRING} --> Using a timestamp {@link String} representation (e.g. {@code
+ *       "2024-10-08T13:18:20.053+03:27"})
+ * 

+ *
+ * 
Note: If you expect Beam to create the Iceberg table at runtime, please provide {@code
+ * SqlTypes.DATETIME} for a {@code timestamp} column and {@code DATETIME} for a {@code timestamptz}
+ * column. If the table does not exist, Beam will treat {@code STRING} and {@code INT64} at
+ * face-value and create equivalent column types.
+ *
+ * 
For Iceberg reads, the connector will produce Beam {@code SqlTypes.DATETIME} types for
+ * Iceberg's {@code timestamp} and {@code DATETIME} types for {@code timestamptz}.
+ *
+ * 
Writing to Tables

+ *
+ * 
Creating Tables

+ *
+ * 
If an Iceberg table does not exist at the time of writing, this connector will automatically
+ * create one with the data's schema.
+ *
+ * 
Note that this is a best-effort operation that depends on the {@link Catalog} implementation.
+ * Some implementations may not support creating a table using the Iceberg API.
+ *
+ * 
Dynamic Destinations

+ *
+ * 
Managed Iceberg supports writing to dynamic destinations. To do so, please provide an
+ * identifier template for the {@code table} parameter. A template should have placeholders
+ * represented as curly braces containing a record field name, e.g.: {@code
+ * "my_namespace.my_{foo}_table"}.
+ *
+ * 
The sink uses simple String interpolation to determine a record's table destination. The
+ * placeholder is replaced with the record's field value. Nested fields can be specified using
+ * dot-notation (e.g. {@code "{top.middle.nested}"}).
+ *
+ * 
Pre-filtering Options

+ *
+ * 
Some use cases may benefit from filtering record fields right before the write operation. For
+ * example, you may want to provide meta-data to guide records to the right destination, but not
+ * necessarily write that meta-data to your table. Some light-weight filtering options are provided
+ * to accommodate such cases, allowing you to control what actually gets written (see {@code
+ * drop}, {@code keep}, {@code only}}).
+ *
+ * 
Example write to dynamic destinations (pseudocode):
+ *
+ * 
{@code
+ * Map config = Map.of(
+ *         "table", "flights.{country}.{airport}",
+ *         "catalog_properties", Map.of(...),
+ *         "drop", ["country", "airport"]);
+ *
+ * JSON_ROWS = [
+ *       // first record is written to table "flights.usa.RDU"
+ *         "{\"country\": \"usa\"," +
+ *          "\"airport\": \"RDU\"," +
+ *          "\"flight_id\": \"AA356\"," +
+ *          "\"destination\": \"SFO\"," +
+ *          "\"meal\": \"chicken alfredo\"}",
+ *       // second record is written to table "flights.qatar.HIA"
+ *         "{\"country\": \"qatar\"," +
+ *          "\"airport\": \"HIA\"," +
+ *          "\"flight_id\": \"QR 875\"," +
+ *          "\"destination\": \"DEL\"," +
+ *          "\"meal\": \"shawarma\"}",
+ *          ...
+ *          ];
+ *
+ * // fields "country" and "airport" are dropped before
+ * // records are written to tables
+ * pipeline
+ *     .apply(Create.of(JSON_ROWS))
+ *     .apply(JsonToRow.withSchema(...))
+ *     .apply(Managed.write(ICEBERG).withConfig(config));
+ *
+ * }

+ *
+ * 
Output Snapshots

+ *
+ * 
When records are written and committed to a table, a snapshot is produced. A batch pipeline
+ * will perform a single commit and create a single snapshot per table. A streaming pipeline will
+ * produce a snapshot roughly according to the configured {@code
+ * triggering_frequency_seconds}.
+ *
+ * 
You can access these snapshots and perform downstream processing by fetching the {@code
+ * "snapshots"} output PCollection:
+ *
+ * 
{@code
+ * pipeline
+ *     .apply(Create.of(BEAM_ROWS))
+ *     .apply(Managed.write(ICEBERG).withConfig(config))
+ *     .get("snapshots")
+ *     .apply(ParDo.of(new DoFn {...});
+ * }

+ *
+ * Each Snapshot is represented as a Beam Row, with the following Schema:
+ *
+ * 
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ *   
+ *     
+ *   
+ * 
+ *     
+ *   
+ *   
+ *     
+ *   
+ * 
+ *     
+ *   
+ *   
+ *     
+ *   
+ * 
 Field   Type   Description 
 {@code table}   {@code str}   Table identifier. 
 {@code manifest_list_location}   {@code str}   Location of the snapshot's manifest list. 
 {@code operation}   {@code str}   Name of the operation that produced the snapshot. 
 {@code parent_id}   {@code long}   The snapshot's parent ID. 
 {@code schema_id}   {@code int}   The id of the schema used when the snapshot was created. 
 {@code summary}   {@code map}   A string map of summary data. 
 {@code timestamp_millis}   {@code long}   The snapshot's timestamp in milliseconds. 

+ *
+ * 

+ * 

+ *
+ * 
Reading from Tables

+ *
+ * With the following configuration,
+ *
+ * 
{@code
+ * Map config = Map.of(
+ *         "table", table,
+ *         "catalog_name", name,
+ *         "catalog_properties", Map.of(...),
+ *         "config_properties", Map.of(...));
+ * }

+ *
+ * Example of a simple batch read:
+ *
+ * 
{@code
+ * PCollection rows = pipeline
+ *     .apply(Managed.read(ICEBERG).withConfig(config))
+ *     .getSinglePCollection();
+ * }

+ *
+ * Example of a simple CDC streaming read:
+ *
+ * 
{@code
+ * PCollection rows = pipeline
+ *     .apply(Managed.read(ICEBERG_CDC).withConfig(config))
+ *     .getSinglePCollection();
+ * }

+ *
+ * 
Note: This reads append-only snapshots. Full CDC is not supported yet.
+ *
+ * 
The CDC streaming source (enabled with {@code streaming=true}) continuously polls the
+ * table for new snapshots, with a default interval of 60 seconds. This can be overridden with
+ * {@code poll_interval_seconds}:
+ *
+ * 
{@code
+ * config.put("streaming", true);
+ * config.put("poll_interval_seconds", 10);
+ * }

+ *
+ * 
Choosing a Starting Point (ICEBERG_CDC only)

+ *
+ * By default, a batch read will start reading from the earliest (oldest) table snapshot. A
+ * streaming read will start reading from the latest (most recent) snapshot. This behavior can be
+ * overridden in a few mutually exclusive ways:
+ *
+ * 
    
+ *   
  • Manually setting a starting strategy with {@code starting_strategy} to be {@code
+ *       "earliest"} or {@code "latest"}.
+ *   
  • Setting a starting snapshot id with {@code from_snapshot}.
+ *   
  • Setting a starting timestamp (milliseconds) with {@code from_timestamp}.
+ * 

+ *
+ * 
For example:
+ *
+ * 
{@code
+ * Map config = Map.of(
+ *         "table", table,
+ *         "catalog_name", name,
+ *         "catalog_properties", Map.of(...),
+ *         "config_properties", Map.of(...),
+ *         "streaming", true,
+ *         "from_snapshot", 123456789L);
+ *
+ * PCollection = pipeline
+ *     .apply(Managed.read(ICEBERG_CDC).withConfig(config))
+ *     .getSinglePCollection();
+ * }

+ *
+ * 
Choosing an End Point (ICEBERG_CDC only)

+ *
+ * By default, a batch read will go up until the most recent table snapshot. A streaming read will
+ * continue monitoring the table for new snapshots forever. This can be overridden with one of the
+ * following options:
+ *
+ * 
    
+ *   
  • Setting an ending snapshot id with {@code to_snapshot}.
+ *   
  • Setting an ending timestamp (milliseconds) with {@code to_timestamp}.
+ * 

+ *
+ * 
For example:
+ *
+ * 
{@code
+ * Map config = Map.of(
+ *         "table", table,
+ *         "catalog_name", name,
+ *         "catalog_properties", Map.of(...),
+ *         "config_properties", Map.of(...),
+ *         "from_snapshot", 123456789L,
+ *         "to_timestamp", 987654321L);
+ *
+ * PCollection = pipeline
+ *     .apply(Managed.read(ICEBERG_CDC).withConfig(config))
+ *     .getSinglePCollection();
+ * }

+ *
+ * Note: If {@code streaming=true} and an end point is set, the pipeline will run in
+ * streaming mode and shut down automatically after processing the final snapshot.
+ */
+@Internal
+public class IcebergIO {
+
+  public static WriteRows writeRows(IcebergCatalogConfig catalog) {
+    return new AutoValue_IcebergIO_WriteRows.Builder()
+        .setCatalogConfig(catalog)
+        .setDistributionMode(DistributionMode.HASH)
+        .setAutoSharding(false)
+        .build();
+  }
+
+  @AutoValue
+  public abstract static class WriteRows extends PTransform, IcebergWriteResult> {
+
+    abstract IcebergCatalogConfig getCatalogConfig();
+
+    abstract @Nullable TableIdentifier getTableIdentifier();
+
+    abstract @Nullable DynamicDestinations getDynamicDestinations();
+
+    abstract @Nullable Duration getTriggeringFrequency();
+
+    abstract @Nullable Integer getDirectWriteByteLimit();
+
+    abstract DistributionMode getDistributionMode();
+
+    abstract @Nullable SerializableFunction getDistributionFunction();
+
+    abstract boolean getAutoSharding();
+
+    abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    abstract static class Builder {
+      abstract Builder setCatalogConfig(IcebergCatalogConfig config);
+
+      abstract Builder setTableIdentifier(TableIdentifier identifier);
+
+      abstract Builder setDynamicDestinations(DynamicDestinations destinations);
+
+      abstract Builder setTriggeringFrequency(Duration triggeringFrequency);
+
+      abstract Builder setDirectWriteByteLimit(Integer directWriteByteLimit);
+
+      abstract Builder setDistributionMode(DistributionMode mode);
+
+      abstract Builder setDistributionFunction(SerializableFunction shardFn);
+
+      abstract Builder setAutoSharding(boolean autoSharding);
+
+      abstract WriteRows build();
+    }
+
+    public WriteRows to(TableIdentifier identifier) {
+      return toBuilder().setTableIdentifier(identifier).build();
+    }
+
+    public WriteRows to(DynamicDestinations destinations) {
+      return toBuilder().setDynamicDestinations(destinations).build();
+    }
+
+    /**
+     * Sets the frequency at which data is written to files and a new {@link
+     * org.apache.iceberg.Snapshot} is produced.
+     *
+     * 
Roughly every triggeringFrequency duration, records are written to data files and appended
+     * to the respective table. Each append operation creates a new table snapshot.
+     *
+     * 
Generally speaking, increasing this duration will result in fewer, larger data files and
+     * fewer snapshots.
+     *
+     * 
This is only applicable when writing an unbounded {@link PCollection} (i.e. a streaming
+     * pipeline).
+     */
+    public WriteRows withTriggeringFrequency(Duration triggeringFrequency) {
+      return toBuilder().setTriggeringFrequency(triggeringFrequency).build();
+    }
+
+    public WriteRows withDirectWriteByteLimit(Integer directWriteByteLimit) {
+      return toBuilder().setDirectWriteByteLimit(directWriteByteLimit).build();
+    }
+
+    /**
+     * Defines the distribution mode of write data prior to writing.
+     *
+     * 
The default distribution mode is {@link DistributionMode#HASH}.
+     *
+     * 
Comparison of Distribution Modes:

+     *
+     * 
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *   
+     * 
Comparison of Distribution Modes
ModeDescriptionProsCons
{@link DistributionMode#NONE}No network shuffle is performed. Records are sorted locally on workers prior to writing.Highly lightweight with zero shuffle/network overhead. Best for smaller data volumes.Writers on different workers can write to overlapping min/max key ranges across multiple files. Relies heavily on post-fact compaction or query time merges.
{@link DistributionMode#HASH}Data is shuffled and consolidated by partition key. All records for a partition are routed to a single worker.Consolidates partition files, eliminating cross-worker file overlapping for partition keys. Excellent worker stability.Can suffer from severe data skew if a single partition contains significantly more data than others (hot partitions).
{@link DistributionMode#RANGE}Data is shuffled based on a user-provided shard/bucket function (e.g., hashing/binning continuous keys).Distributes writes for hot partitions across multiple workers. Eliminates skew while keeping file min/max key ranges tight and non-overlapping.Requires providing a custom {@link SerializableFunction} mapping rows to integer shard/bucket IDs.

+     *
+     * 
Recommendation Matrix (Sorting & Partitioning vs. Scale):

+     *
+     * 
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     *   
+     *     
+     *     
+     *     
+     *     
+     *     
+     *     
+     *   
+     * 
Recommendation Matrix
PartitioningSortingScale / VolumeLatency PriorityRecommended ModeOperational Impact
PartitionedSortedSmallAny{@link DistributionMode#HASH}Consolidates partition files and sorts them locally. Avoids file overlaps for small volumes.
PartitionedSortedMedium / LargeLow Write Latency{@link DistributionMode#NONE}Eliminates shuffle overhead for maximum write speed. Results in overlapping key ranges across files, which requires downstream compaction.
PartitionedSortedMedium / LargeLow Read Latency{@link DistributionMode#HASH} with auto-sharding OR {@link DistributionMode#RANGE}HASH with auto-sharding scales writes for hot partitions but can result in overlapping file ranges requiring query-time sort merges. RANGE sharding distributes hot partitions into sequential, non-overlapping files to optimize reads.
PartitionedUnsortedSmallAny{@link DistributionMode#HASH}Consolidates data files into single partition directories to prevent file fragmentation.
PartitionedUnsortedMedium / LargeAny{@link DistributionMode#HASH} with auto-shardingConsolidates partition files while dynamically balancing hot partition writes across parallel workers.
UnpartitionedSortedSmallAny{@link DistributionMode#NONE}Bypasses network shuffle for fast, low-volume local sorting.
UnpartitionedSortedMedium / LargeLow Write Latency{@link DistributionMode#NONE}Bypasses network shuffle for parallel worker writes. Requires downstream compaction to resolve overlapping file ranges.
UnpartitionedSortedMedium / LargeLow Read Latency{@link DistributionMode#RANGE} (with custom sharding function)Shards continuous keys into non-overlapping worker ranges. Eliminates single-worker bottlenecks and guarantees zero file overlap for fast queries.
UnpartitionedUnsortedAnyAny{@link DistributionMode#NONE}Direct, parallel worker writes with maximum throughput and zero network shuffle overhead.

+     *
+     * 
Code Samples:

+     *
+     * 
{@code
+     * // 1. Using default HASH distribution mode (Consolidates by partition key)
+     * pipeline
+     *     .apply(Create.of(BEAM_ROWS))
+     *     .apply(IcebergIO.writeRows(catalogConfig)
+     *         .to(tableId));
+     *
+     * // 2. Using NONE distribution mode (No shuffle, local sorting only)
+     * pipeline
+     *     .apply(Create.of(BEAM_ROWS))
+     *     .apply(IcebergIO.writeRows(catalogConfig)
+     *         .to(tableId)
+     *         .withDistributionMode(DistributionMode.NONE));
+     *
+     * // 3. Using RANGE distribution mode with a custom shard/bucket function to avoid data skew
+     * pipeline
+     *     .apply(Create.of(BEAM_ROWS))
+     *     .apply(IcebergIO.writeRows(catalogConfig)
+     *         .to(tableId)
+     *         .withDistributionMode(DistributionMode.RANGE)
+     *         .withDistributionFunction(row -> {
+     *             // Group continuous IDs into 16 parallel, non-overlapping shards
+     *             long id = row.getInt64("id");
+     *             return (int) (id / 10000);
+     *         }));
+     * }

+     *
+     * @param mode The distribution mode.
+     */
+    public WriteRows withDistributionMode(DistributionMode mode) {
+      return toBuilder().setDistributionMode(mode).build();
+    }
+
+    /**
+     * Sets the custom range-distribution function.
+     *
+     * 
Only applicable when the distribution mode is set to {@link DistributionMode#RANGE}. The
+     * function maps a Beam {@link Row} to an Integer representing a shard/bucket ID.
+     */
+    public WriteRows withDistributionFunction(SerializableFunction shardFn) {
+      return toBuilder().setDistributionFunction(shardFn).build();
+    }
+
+    /**
+     * Enables Beam's dynamic auto-sharding when using {@link DistributionMode#HASH}.
+     *
+     * 
When enabled, the pipeline uses {@link
+     * org.apache.beam.sdk.transforms.GroupIntoBatches#withShardedKey()} under the hood. The runner
+     * (such as Dataflow) dynamically monitors throughput per partition key. If a partition is
+     * extremely hot, the runner automatically splits it into parallel sub-shards distributed across
+     * multiple workers to prevent single-worker bottlenecks and out-of-memory (OOM) errors, while
+     * keeping the number of data files for cold partitions minimal.
+     *
+     * 
Note that because auto-sharding distributes hot-partition data randomly across worker
+     * shards, the written data files cannot guarantee non-overlapping key ranges. Downstream
+     * queries may require read-time sort merges for overlapping file segments until an Iceberg
+     * compaction job (e.g., `rewriteDataFiles`) is executed.
+     *
+     * 
Only applicable when using {@link DistributionMode#HASH}.
+     */
+    public WriteRows withAutosharding() {
+      return toBuilder().setAutoSharding(true).build();
+    }
+
+    @Override
+    public IcebergWriteResult expand(PCollection input) {
+      List allToArgs = Arrays.asList(getTableIdentifier(), getDynamicDestinations());
+      Preconditions.checkArgument(
+          1 == allToArgs.stream().filter(Predicates.notNull()).count(),
+          "Must set exactly one of table identifier or dynamic destinations object.");
+
+      DynamicDestinations destinations = getDynamicDestinations();
+      if (destinations == null) {
+        destinations =
+            DynamicDestinations.singleTable(
+                Preconditions.checkNotNull(getTableIdentifier()), input.getSchema());
+      }
+
+      // Assign destinations before re-windowing to global in WriteToDestinations because
+      // user's dynamic destination may depend on windowing properties
+      if (IcebergUtils.validDirectWriteLimit(getDirectWriteByteLimit())) {
+        Preconditions.checkArgument(
+            IcebergUtils.isUnbounded(input),
+            "Must only provide direct write limit for unbounded pipelines.");
+      }
+
+      switch (getDistributionMode()) {
+        case NONE:
+          Preconditions.checkArgument(
+              !getAutoSharding(),
+              "Autosharding option is only available with " + "'hash' distribution mode.");
+          return input
+              .apply("Assign Table Destinations", new AssignDestinations(destinations))
+              .apply(
+                  "Write Rows to Destinations",
+                  new WriteToDestinations(
+                      getCatalogConfig(),
+                      destinations,
+                      getTriggeringFrequency(),
+                      getDirectWriteByteLimit()));
+        case HASH:
+          return input
+              .apply(
+                  "AssignDestinationAndPartition",
+                  new AssignDestinationsAndPartitions(
+                      destinations,
+                      getCatalogConfig(),
+                      getDistributionMode(),
+                      getDistributionFunction()))
+              .apply(
+                  "Write Rows to Partitions",
+                  new WriteToPartitions(
+                      getCatalogConfig(),
+                      destinations,
+                      getTriggeringFrequency(),
+                      getAutoSharding()));
+        case RANGE:
+          Preconditions.checkArgument(
+              getDistributionFunction() != null,
+              "Must provide a distribution function when using RANGE distribution mode.");
+          return input
+              .apply(
+                  "AssignDestinationAndPartitionWithRange",
+                  new AssignDestinationsAndPartitions(
+                      destinations,
+                      getCatalogConfig(),
+                      getDistributionMode(),
+                      getDistributionFunction()))
+              .apply(
+                  "Write Rows to Partitions",
+                  new WriteToPartitions(
+                      getCatalogConfig(),
+                      destinations,
+                      getTriggeringFrequency(),
+                      getAutoSharding()));
+        default:
+          throw new UnsupportedOperationException(
+              "Unsupported distribution mode: " + getDistributionMode());
+      }
+    }
+  }
+
+  public static ReadRows readRows(IcebergCatalogConfig catalogConfig) {
+    return new AutoValue_IcebergIO_ReadRows.Builder()
+        .setCatalogConfig(catalogConfig)
+        .setUseCdc(false)
+        .build();
+  }
+
+  @AutoValue
+  public abstract static class ReadRows extends PTransform> {
+    public enum StartingStrategy {
+      EARLIEST,
+      LATEST
+    }
+
+    abstract IcebergCatalogConfig getCatalogConfig();
+
+    abstract @Nullable TableIdentifier getTableIdentifier();
+
+    abstract boolean getUseCdc();
+
+    abstract @Nullable Long getFromSnapshot();
+
+    abstract @Nullable Long getToSnapshot();
+
+    abstract @Nullable Long getFromTimestamp();
+
+    abstract @Nullable Long getToTimestamp();
+
+    abstract @Nullable StartingStrategy getStartingStrategy();
+
+    abstract @Nullable Boolean getStreaming();
+
+    abstract @Nullable Duration getPollInterval();
+
+    abstract @Nullable List getKeep();
+
+    abstract @Nullable List getDrop();
+
+    abstract @Nullable String getFilter();
+
+    abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    abstract static class Builder {
+      abstract Builder setCatalogConfig(IcebergCatalogConfig config);
+
+      abstract Builder setTableIdentifier(TableIdentifier identifier);
+
+      abstract Builder setUseCdc(boolean useCdc);
+
+      abstract Builder setFromSnapshot(@Nullable Long fromSnapshot);
+
+      abstract Builder setToSnapshot(@Nullable Long toSnapshot);
+
+      abstract Builder setFromTimestamp(@Nullable Long fromTimestamp);
+
+      abstract Builder setToTimestamp(@Nullable Long toTimestamp);
+
+      abstract Builder setStartingStrategy(@Nullable StartingStrategy strategy);
+
+      abstract Builder setStreaming(@Nullable Boolean streaming);
+
+      abstract Builder setPollInterval(@Nullable Duration triggeringFrequency);
+
+      abstract Builder setKeep(@Nullable List fields);
+
+      abstract Builder setDrop(@Nullable List fields);
+
+      abstract Builder setFilter(@Nullable String filter);
+
+      abstract ReadRows build();
+    }
+
+    public ReadRows withCdc() {
+      return toBuilder().setUseCdc(true).build();
+    }
+
+    public ReadRows from(TableIdentifier tableIdentifier) {
+      return toBuilder().setTableIdentifier(tableIdentifier).build();
+    }
+
+    public ReadRows fromSnapshot(@Nullable Long fromSnapshot) {
+      return toBuilder().setFromSnapshot(fromSnapshot).build();
+    }
+
+    public ReadRows toSnapshot(@Nullable Long toSnapshot) {
+      return toBuilder().setToSnapshot(toSnapshot).build();
+    }
+
+    public ReadRows fromTimestamp(@Nullable Long fromTimestamp) {
+      return toBuilder().setFromTimestamp(fromTimestamp).build();
+    }
+
+    public ReadRows toTimestamp(@Nullable Long toTimestamp) {
+      return toBuilder().setToTimestamp(toTimestamp).build();
+    }
+
+    public ReadRows withPollInterval(Duration pollInterval) {
+      return toBuilder().setPollInterval(pollInterval).build();
+    }
+
+    public ReadRows streaming(@Nullable Boolean streaming) {
+      return toBuilder().setStreaming(streaming).build();
+    }
+
+    public ReadRows withStartingStrategy(@Nullable StartingStrategy strategy) {
+      return toBuilder().setStartingStrategy(strategy).build();
+    }
+
+    public ReadRows keeping(@Nullable List keep) {
+      return toBuilder().setKeep(keep).build();
+    }
+
+    public ReadRows dropping(@Nullable List drop) {
+      return toBuilder().setDrop(drop).build();
+    }
+
+    public ReadRows withFilter(@Nullable String filter) {
+      return toBuilder().setFilter(filter).build();
+    }
+
+    @Override
+    public PCollection expand(PBegin input) {
+      TableIdentifier tableId =
+          checkStateNotNull(getTableIdentifier(), "Must set a table to read from.");
+
+      Table table = getCatalogConfig().catalog().loadTable(tableId);
+
+      IcebergScanConfig scanConfig =
+          IcebergScanConfig.builder()
+              .setCatalogConfig(getCatalogConfig())
+              .setScanType(IcebergScanConfig.ScanType.TABLE)
+              .setTableIdentifier(tableId)
+              .setSchema(IcebergUtils.icebergSchemaToBeamSchema(table.schema()))
+              .setFromSnapshotInclusive(getFromSnapshot())
+              .setToSnapshot(getToSnapshot())
+              .setFromTimestamp(getFromTimestamp())
+              .setToTimestamp(getToTimestamp())
+              .setStartingStrategy(getStartingStrategy())
+              .setStreaming(getStreaming())
+              .setPollInterval(getPollInterval())
+              .setUseCdc(getUseCdc())
+              .setKeepFields(getKeep())
+              .setDropFields(getDrop())
+              .setFilterString(getFilter())
+              .build();
+      scanConfig.validate(table);
+
+      PTransform> source =
+          getUseCdc()
+              ? new IncrementalScanSource(scanConfig)
+              : Read.from(new ScanSource(scanConfig));
+
+      return input.apply(source);
+    }
+  }
+}
diff --git a/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java
new file mode 100644
index 000000000000..6efc1bbe2eec
--- /dev/null
+++ b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergRowSorter.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.beam.sdk.coders.RowCoder;
+import org.apache.beam.sdk.extensions.sorter.BufferedExternalSorter;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.Row;
+import org.apache.iceberg.NullOrder;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortDirection;
+import org.apache.iceberg.SortField;
+import org.apache.iceberg.SortOrder;
+import org.joda.time.ReadableInstant;
+
+/**
+ * A utility class to sort Beam {@link Row}s based on an Iceberg {@link SortOrder}. Leverages {@link
+ * BufferedExternalSorter} to spill to local disk when elements exceed memory limit.
+ */
+class IcebergRowSorter implements Serializable {
+
+  public static Iterable sortRows(
+      Iterable rows,
+      SortOrder sortOrder,
+      Schema icebergSchema,
+      org.apache.beam.sdk.schemas.Schema beamSchema) {
+
+    if (sortOrder == null || !sortOrder.isSorted()) {
+      return rows;
+    }
+
+    BufferedExternalSorter.Options sorterOptions = BufferedExternalSorter.options();
+    BufferedExternalSorter sorter = BufferedExternalSorter.create(sorterOptions);
+    RowCoder rowCoder = RowCoder.of(beamSchema);
+
+    List fields = sortOrder.fields();
+    String[] columnNames = new String[fields.size()];
+    for (int i = 0; i < fields.size(); i++) {
+      columnNames[i] = icebergSchema.findColumnName(fields.get(i).sourceId());
+    }
+
+    // Create reusable ByteArrayOutputStreams for key and value encoding
+    ByteArrayOutputStream keyBaos = new ByteArrayOutputStream();
+    ByteArrayOutputStream valBaos = new ByteArrayOutputStream();
+
+    try {
+      for (Row row : rows) {
+        keyBaos.reset();
+        valBaos.reset();
+        encodeSortKey(row, sortOrder, columnNames, keyBaos, icebergSchema, beamSchema);
+        byte[] keyBytes = keyBaos.toByteArray();
+
+        rowCoder.encode(row, valBaos);
+        byte[] valBytes = valBaos.toByteArray();
+        sorter.add(KV.of(keyBytes, valBytes));
+      }
+
+      Iterable> sortedKVs = sorter.sort();
+      return new Iterable() {
+        @Override
+        public Iterator iterator() {
+          final Iterator> it = sortedKVs.iterator();
+          return new Iterator() {
+            @Override
+            public boolean hasNext() {
+              return it.hasNext();
+            }
+
+            @Override
+            public Row next() {
+              KV next = it.next();
+              try {
+                ByteArrayInputStream bais = new ByteArrayInputStream(next.getValue());
+                return rowCoder.decode(bais);
+              } catch (IOException e) {
+                throw new RuntimeException("Failed to decode Row during sorting", e);
+              }
+            }
+          };
+        }
+      };
+
+    } catch (IOException e) {
+      throw new RuntimeException("Failed to sort rows with external sorter", e);
+    }
+  }
+
+  @SuppressWarnings("nullness")
+  public static void encodeSortKey(
+      Row row,
+      SortOrder sortOrder,
+      String[] columnNames,
+      ByteArrayOutputStream baos,
+      Schema icebergSchema,
+      org.apache.beam.sdk.schemas.Schema beamSchema)
+      throws IOException {
+
+    List fields = sortOrder.fields();
+
+    for (int i = 0; i < fields.size(); i++) {
+      SortField field = fields.get(i);
+      String colName = columnNames[i];
+      Object val = row.getValue(colName);
+
+      if (!field.transform().isIdentity()) {
+        Object icebergVal =
+            IcebergUtils.beamValueToIcebergValue(icebergSchema.findType(field.sourceId()), val);
+        if (icebergVal != null) {
+          val = field.transform().apply(icebergVal);
+        } else {
+          val = null;
+        }
+      }
+
+      boolean isNull = (val == null);
+      boolean isDesc = (field.direction() == SortDirection.DESC);
+      boolean nullsFirst = (field.nullOrder() == NullOrder.NULLS_FIRST);
+
+      // Determine correct header prefix to fulfill the NullOrder contracts
+      byte prefixByte;
+      if (isNull) {
+        prefixByte = nullsFirst ? (byte) 0x00 : (byte) 0xFF;
+      } else {
+        prefixByte = nullsFirst ? (byte) 0x01 : (byte) 0x00;
+      }
+
+      baos.write(prefixByte);
+
+      if (!isNull) {
+        writeValue(val, baos, isDesc);
+      }
+    }
+  }
+
+  private static void writeInt(int v, ByteArrayOutputStream baos, boolean invert) {
+    byte b3 = (byte) (v >>> 24);
+    byte b2 = (byte) (v >>> 16);
+    byte b1 = (byte) (v >>> 8);
+    byte b0 = (byte) v;
+    if (invert) {
+      baos.write(~b3);
+      baos.write(~b2);
+      baos.write(~b1);
+      baos.write(~b0);
+    } else {
+      baos.write(b3);
+      baos.write(b2);
+      baos.write(b1);
+      baos.write(b0);
+    }
+  }
+
+  private static void writeLong(long v, ByteArrayOutputStream baos, boolean invert) {
+    byte b7 = (byte) (v >>> 56);
+    byte b6 = (byte) (v >>> 48);
+    byte b5 = (byte) (v >>> 40);
+    byte b4 = (byte) (v >>> 32);
+    byte b3 = (byte) (v >>> 24);
+    byte b2 = (byte) (v >>> 16);
+    byte b1 = (byte) (v >>> 8);
+    byte b0 = (byte) v;
+    if (invert) {
+      baos.write(~b7);
+      baos.write(~b6);
+      baos.write(~b5);
+      baos.write(~b4);
+      baos.write(~b3);
+      baos.write(~b2);
+      baos.write(~b1);
+      baos.write(~b0);
+    } else {
+      baos.write(b7);
+      baos.write(b6);
+      baos.write(b5);
+      baos.write(b4);
+      baos.write(b3);
+      baos.write(b2);
+      baos.write(b1);
+      baos.write(b0);
+    }
+  }
+
+  @SuppressWarnings("JavaUtilDate")
+  private static void writeValue(Object val, ByteArrayOutputStream baos, boolean invert)
+      throws IOException {
+    if (val instanceof String) {
+      writeString((String) val, baos, invert);
+    } else if (val instanceof Integer) {
+      int v = (Integer) val;
+      writeInt(v ^ Integer.MIN_VALUE, baos, invert);
+    } else if (val instanceof Long) {
+      long v = (Long) val;
+      writeLong(v ^ Long.MIN_VALUE, baos, invert);
+    } else if (val instanceof Float) {
+      int bits = Float.floatToIntBits((Float) val);
+      bits = (bits >= 0) ? (bits ^ Integer.MIN_VALUE) : ~bits;
+      writeInt(bits, baos, invert);
+    } else if (val instanceof Double) {
+      long bits = Double.doubleToLongBits((Double) val);
+      bits = (bits >= 0) ? (bits ^ Long.MIN_VALUE) : ~bits;
+      writeLong(bits, baos, invert);
+    } else if (val instanceof Boolean) {
+      byte b = ((Boolean) val) ? (byte) 0x01 : (byte) 0x00;
+      baos.write(invert ? ~b : b);
+    } else if (val instanceof byte[]) {
+      writeByteArray((byte[]) val, baos, invert);
+    } else if (val instanceof ByteBuffer) {
+      writeByteArray(((ByteBuffer) val).array(), baos, invert);
+    } else if (val instanceof ReadableInstant) {
+      long enc = ((ReadableInstant) val).getMillis() ^ Long.MIN_VALUE;
+      writeLong(enc, baos, invert);
+    } else if (val instanceof Instant) {
+      long enc = ((Instant) val).toEpochMilli() ^ Long.MIN_VALUE;
+      writeLong(enc, baos, invert);
+    } else if (val instanceof Date) {
+      long enc = ((Date) val).getTime() ^ Long.MIN_VALUE;
+      writeLong(enc, baos, invert);
+    } else {
+      throw new UnsupportedOperationException(
+          "Unsupported type for sorting: " + val.getClass().getName());
+    }
+  }
+
+  private static void writeString(String s, ByteArrayOutputStream baos, boolean invert)
+      throws IOException {
+    byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+    writeByteArray(bytes, baos, invert);
+  }
+
+  private static void writeByteArray(byte[] bytes, ByteArrayOutputStream baos, boolean invert) {
+    for (byte b : bytes) {
+      if (b == 0x00) {
+        baos.write(invert ? ~(byte) 0x01 : (byte) 0x01);
+        baos.write(invert ? ~(byte) 0x01 : (byte) 0x01);
+      } else if (b == 0x01) {
+        baos.write(invert ? ~(byte) 0x01 : (byte) 0x01);
+        baos.write(invert ? ~(byte) 0x02 : (byte) 0x02);
+      } else {
+        baos.write(invert ? ~b : b);
+      }
+    }
+    baos.write(invert ? ~(byte) 0x00 : (byte) 0x00);
+  }
+}
diff --git a/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
new file mode 100644
index 000000000000..7f48a0d0128c
--- /dev/null
+++ b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
@@ -0,0 +1,683 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg;
+
+import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
+import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
+import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState;
+
+import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.logicaltypes.FixedPrecisionNumeric;
+import org.apache.beam.sdk.schemas.logicaltypes.MicrosInstant;
+import org.apache.beam.sdk.schemas.logicaltypes.PassThroughLogicalType;
+import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
+import org.apache.beam.sdk.util.Preconditions;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+
+/** Utilities for converting between Beam and Iceberg types, made public for user's convenience. */
+public class IcebergUtils {
+  private IcebergUtils() {}
+
+  private static final Map BEAM_TYPES_TO_ICEBERG_TYPES =
+      ImmutableMap.builder()
+          .put(Schema.TypeName.BOOLEAN, Types.BooleanType.get())
+          .put(Schema.TypeName.INT32, Types.IntegerType.get())
+          .put(Schema.TypeName.INT64, Types.LongType.get())
+          .put(Schema.TypeName.FLOAT, Types.FloatType.get())
+          .put(Schema.TypeName.DOUBLE, Types.DoubleType.get())
+          .put(Schema.TypeName.STRING, Types.StringType.get())
+          .put(Schema.TypeName.BYTES, Types.BinaryType.get())
+          .put(Schema.TypeName.DATETIME, Types.TimestampType.withZone())
+          .build();
+
+  private static final Map BEAM_LOGICAL_TYPES_TO_ICEBERG_TYPES =
+      ImmutableMap.builder()
+          .put(SqlTypes.DATE.getIdentifier(), Types.DateType.get())
+          .put(SqlTypes.TIME.getIdentifier(), Types.TimeType.get())
+          .put(SqlTypes.DATETIME.getIdentifier(), Types.TimestampType.withoutZone())
+          .put(SqlTypes.UUID.getIdentifier(), Types.UUIDType.get())
+          .put(MicrosInstant.IDENTIFIER, Types.TimestampType.withZone())
+          .build();
+
+  private static Schema.FieldType icebergTypeToBeamFieldType(final Type type) {
+    switch (type.typeId()) {
+      case BOOLEAN:
+        return Schema.FieldType.BOOLEAN;
+      case INTEGER:
+        return Schema.FieldType.INT32;
+      case LONG:
+        return Schema.FieldType.INT64;
+      case FLOAT:
+        return Schema.FieldType.FLOAT;
+      case DOUBLE:
+        return Schema.FieldType.DOUBLE;
+      case DATE:
+        return Schema.FieldType.logicalType(SqlTypes.DATE);
+      case TIME:
+        return Schema.FieldType.logicalType(SqlTypes.TIME);
+      case TIMESTAMP:
+        Types.TimestampType ts = (Types.TimestampType) type.asPrimitiveType();
+        if (ts.shouldAdjustToUTC()) {
+          return Schema.FieldType.DATETIME;
+        }
+        return Schema.FieldType.logicalType(SqlTypes.DATETIME);
+      case STRING:
+        return Schema.FieldType.STRING;
+      case UUID:
+      case BINARY:
+        return Schema.FieldType.BYTES;
+      case FIXED:
+      case DECIMAL:
+        return Schema.FieldType.DECIMAL;
+      case STRUCT:
+        return Schema.FieldType.row(icebergStructTypeToBeamSchema(type.asStructType()));
+      case LIST:
+        return Schema.FieldType.array(icebergTypeToBeamFieldType(type.asListType().elementType()));
+      case MAP:
+        return Schema.FieldType.map(
+            icebergTypeToBeamFieldType(type.asMapType().keyType()),
+            icebergTypeToBeamFieldType(type.asMapType().valueType()));
+      default:
+        throw new RuntimeException("Unrecognized Iceberg Type: " + type.typeId());
+    }
+  }
+
+  private static Schema.Field icebergFieldToBeamField(final Types.NestedField field) {
+    return Schema.Field.of(field.name(), icebergTypeToBeamFieldType(field.type()))
+        .withNullable(field.isOptional());
+  }
+
+  /** Converts an Iceberg {@link org.apache.iceberg.Schema} to a Beam {@link Schema}. */
+  public static Schema icebergSchemaToBeamSchema(final org.apache.iceberg.Schema schema) {
+    Schema.Builder builder = Schema.builder();
+    for (Types.NestedField f : schema.columns()) {
+      builder.addField(icebergFieldToBeamField(f));
+    }
+    return builder.build();
+  }
+
+  private static Schema icebergStructTypeToBeamSchema(final Types.StructType struct) {
+    Schema.Builder builder = Schema.builder();
+    for (Types.NestedField f : struct.fields()) {
+      builder.addField(icebergFieldToBeamField(f));
+    }
+    return builder.build();
+  }
+
+  /**
+   * Represents a {@link Type} and the most recent field ID used to build it.
+   *
+   * 
Iceberg Schema fields are required to have unique IDs. This includes unique IDs for a {@link
+   * org.apache.iceberg.types.Type.NestedType}'s components (e.g. {@link Types.ListType}'s
+   * collection type, {@link Types.MapType}'s key type and value type, and {@link
+   * Types.StructType}'s nested fields). The {@code maxId} in this object represents the most recent
+   * ID used after building this type. This helps signal that the next {@link
+   * org.apache.iceberg.types.Type.NestedType} we construct should have an ID greater than this one.
+   */
+  @VisibleForTesting
+  static class TypeAndMaxId {
+    int maxId;
+    Type type;
+
+    TypeAndMaxId(int id, Type object) {
+      this.maxId = id;
+      this.type = object;
+    }
+  }
+
+  /**
+   * Takes a Beam {@link Schema.FieldType} and an index intended as a starting point for Iceberg
+   * {@link org.apache.iceberg.types.Type.NestedType}s. Returns an Iceberg {@link Type} and the
+   * maximum index after building that type.
+   *
+   * 
Returns this information in an {@link TypeAndMaxId} object.
+   */
+  @VisibleForTesting
+  static TypeAndMaxId beamFieldTypeToIcebergFieldType(
+      Schema.FieldType beamType, int nestedFieldId) {
+    if (BEAM_TYPES_TO_ICEBERG_TYPES.containsKey(beamType.getTypeName())) {
+      // we don't use nested field ID for primitive types. decrement it so the caller can use it for
+      // other types.
+      return new TypeAndMaxId(
+          --nestedFieldId, BEAM_TYPES_TO_ICEBERG_TYPES.get(beamType.getTypeName()));
+    } else if (beamType.getTypeName().isLogicalType()) {
+      Schema.LogicalType logicalType = checkArgumentNotNull(beamType.getLogicalType());
+      if (logicalType instanceof FixedPrecisionNumeric) {
+        Row args = Preconditions.checkArgumentNotNull(logicalType.getArgument());
+        Integer precision = Preconditions.checkArgumentNotNull(args.getInt32("precision"));
+        Integer scale = Preconditions.checkArgumentNotNull(args.getInt32("scale"));
+        return new TypeAndMaxId(--nestedFieldId, Types.DecimalType.of(precision, scale));
+      }
+      if (logicalType instanceof PassThroughLogicalType) {
+        return beamFieldTypeToIcebergFieldType(logicalType.getBaseType(), nestedFieldId);
+      }
+      String logicalTypeIdentifier = logicalType.getIdentifier();
+      @Nullable Type type = BEAM_LOGICAL_TYPES_TO_ICEBERG_TYPES.get(logicalTypeIdentifier);
+      if (type == null) {
+        throw new RuntimeException("Unsupported Beam logical type " + logicalTypeIdentifier);
+      }
+      return new TypeAndMaxId(--nestedFieldId, type);
+    } else if (beamType.getTypeName().isCollectionType()) { // ARRAY or ITERABLE
+      Schema.FieldType beamCollectionType =
+          Preconditions.checkArgumentNotNull(beamType.getCollectionElementType());
+
+      // nestedFieldId is reserved for the list's collection type.
+      // we increment here because further nested fields should use unique ID's
+      TypeAndMaxId listInfo =
+          beamFieldTypeToIcebergFieldType(beamCollectionType, nestedFieldId + 1);
+      Type icebergCollectionType = listInfo.type;
+
+      boolean elementTypeIsNullable =
+          Preconditions.checkArgumentNotNull(beamType.getCollectionElementType()).getNullable();
+
+      Type listType =
+          elementTypeIsNullable
+              ? Types.ListType.ofOptional(nestedFieldId, icebergCollectionType)
+              : Types.ListType.ofRequired(nestedFieldId, icebergCollectionType);
+
+      return new TypeAndMaxId(listInfo.maxId, listType);
+    } else if (beamType.getTypeName().isMapType()) { // MAP
+      // key and value IDs need to be unique
+      int keyId = nestedFieldId;
+      int valueId = keyId + 1;
+
+      // nested field IDs should be unique
+      nestedFieldId = valueId + 1;
+      Schema.FieldType beamKeyType = Preconditions.checkArgumentNotNull(beamType.getMapKeyType());
+      TypeAndMaxId keyInfo = beamFieldTypeToIcebergFieldType(beamKeyType, nestedFieldId);
+      Type icebergKeyType = keyInfo.type;
+
+      nestedFieldId = keyInfo.maxId + 1;
+      Schema.FieldType beamValueType =
+          Preconditions.checkArgumentNotNull(beamType.getMapValueType());
+      TypeAndMaxId valueInfo = beamFieldTypeToIcebergFieldType(beamValueType, nestedFieldId);
+      Type icebergValueType = valueInfo.type;
+
+      Type mapType =
+          beamValueType.getNullable()
+              ? Types.MapType.ofOptional(keyId, valueId, icebergKeyType, icebergValueType)
+              : Types.MapType.ofRequired(keyId, valueId, icebergKeyType, icebergValueType);
+
+      return new TypeAndMaxId(valueInfo.maxId, mapType);
+    } else if (beamType.getTypeName().isCompositeType()) { // ROW
+      // Nested field IDs need to be unique from the field that contains this StructType
+      Schema nestedSchema = Preconditions.checkArgumentNotNull(beamType.getRowSchema());
+      List nestedFields = new ArrayList<>(nestedSchema.getFieldCount());
+
+      int icebergFieldId = nestedFieldId;
+      nestedFieldId = icebergFieldId + nestedSchema.getFieldCount();
+      for (Schema.Field beamField : nestedSchema.getFields()) {
+        TypeAndMaxId typeAndMaxId =
+            beamFieldTypeToIcebergFieldType(beamField.getType(), nestedFieldId);
+        Types.NestedField icebergField =
+            Types.NestedField.of(
+                icebergFieldId++,
+                beamField.getType().getNullable(),
+                beamField.getName(),
+                typeAndMaxId.type);
+
+        nestedFields.add(icebergField);
+        nestedFieldId = typeAndMaxId.maxId + 1;
+      }
+
+      Type structType = Types.StructType.of(nestedFields);
+
+      return new TypeAndMaxId(nestedFieldId - 1, structType);
+    }
+
+    return new TypeAndMaxId(nestedFieldId, Types.StringType.get());
+  }
+
+  /**
+   * Converts a Beam {@link Schema} to an Iceberg {@link org.apache.iceberg.Schema}.
+   *
+   * 
The following unsupported Beam types will be defaulted to {@link Types.StringType}:
+   * 
  • {@link Schema.TypeName.DECIMAL}
+   */
+  public static org.apache.iceberg.Schema beamSchemaToIcebergSchema(final Schema schema) {
+    List fields = new ArrayList<>(schema.getFieldCount());
+    int nestedFieldId = schema.getFieldCount() + 1;
+    int icebergFieldId = 1;
+    for (Schema.Field beamField : schema.getFields()) {
+      TypeAndMaxId typeAndMaxId =
+          beamFieldTypeToIcebergFieldType(beamField.getType(), nestedFieldId);
+      Types.NestedField icebergField =
+          Types.NestedField.of(
+              icebergFieldId++,
+              beamField.getType().getNullable(),
+              beamField.getName(),
+              typeAndMaxId.type);
+
+      fields.add(icebergField);
+      nestedFieldId = typeAndMaxId.maxId + 1;
+    }
+    return new org.apache.iceberg.Schema(fields.toArray(new Types.NestedField[fields.size()]));
+  }
+
+  /**
+   * Converts a Beam field value to its Iceberg-compatible equivalent based on the Iceberg {@link
+   * Type}.
+   */
+  public static @Nullable Object beamValueToIcebergValue(Type type, @Nullable Object value) {
+    if (value == null) {
+      return null;
+    }
+    switch (type.typeId()) {
+      case BOOLEAN:
+      case INTEGER:
+      case LONG:
+      case FLOAT:
+      case DOUBLE:
+      case DATE:
+      case TIME:
+      case DECIMAL:
+      case STRING:
+        return value;
+      case TIMESTAMP:
+        Types.TimestampType ts = (Types.TimestampType) type.asPrimitiveType();
+        return getIcebergTimestampValue(value, ts.shouldAdjustToUTC());
+      case UUID:
+        if (value instanceof byte[]) {
+          return UUID.nameUUIDFromBytes((byte[]) value);
+        }
+        return value;
+      case BINARY:
+        if (value instanceof byte[]) {
+          return ByteBuffer.wrap((byte[]) value);
+        }
+        return value;
+      case FIXED:
+        throw new UnsupportedOperationException("Fixed-precision fields are not yet supported.");
+      default:
+        return value;
+    }
+  }
+
+  /** Converts a Beam {@link Row} to an Iceberg {@link Record}. */
+  public static Record beamRowToIcebergRecord(org.apache.iceberg.Schema schema, Row row) {
+    if (row.getSchema().getFieldCount() != schema.columns().size()) {
+      throw new IllegalStateException(
+          String.format(
+              "Beam Row schema and Iceberg schema have different sizes.%n\tBeam Row columns: %s%n\tIceberg schema columns: %s",
+              row.getSchema().getFieldNames(),
+              schema.columns().stream().map(Types.NestedField::name).collect(Collectors.toList())));
+    }
+    return copyRowIntoRecord(GenericRecord.create(schema), row);
+  }
+
+  private static Record copyRowIntoRecord(Record baseRecord, Row value) {
+    Record rec = baseRecord.copy();
+    for (Types.NestedField f : rec.struct().fields()) {
+      copyFieldIntoRecord(rec, f, value);
+    }
+    return rec;
+  }
+
+  private static void copyFieldIntoRecord(Record rec, Types.NestedField field, Row value) {
+    String name = field.name();
+    switch (field.type().typeId()) {
+      case BOOLEAN:
+        Optional.ofNullable(value.getBoolean(name)).ifPresent(v -> rec.setField(name, v));
+        break;
+      case INTEGER:
+        Optional.ofNullable(value.getInt32(name)).ifPresent(v -> rec.setField(name, v));
+        break;
+      case LONG:
+        Optional.ofNullable(value.getInt64(name)).ifPresent(v -> rec.setField(name, v));
+        break;
+      case FLOAT:
+        Optional.ofNullable(value.getFloat(name)).ifPresent(v -> rec.setField(name, v));
+        break;
+      case DOUBLE:
+        Optional.ofNullable(value.getDouble(name)).ifPresent(v -> rec.setField(name, v));
+        break;
+      case DATE:
+        Optional.ofNullable(value.getLogicalTypeValue(name, LocalDate.class))
+            .ifPresent(v -> rec.setField(name, v));
+        break;
+      case TIME:
+        Optional.ofNullable(value.getLogicalTypeValue(name, LocalTime.class))
+            .ifPresent(v -> rec.setField(name, v));
+        break;
+      case TIMESTAMP:
+        Object val = value.getValue(name);
+        if (val == null) {
+          break;
+        }
+        Types.TimestampType ts = (Types.TimestampType) field.type().asPrimitiveType();
+        rec.setField(name, getIcebergTimestampValue(val, ts.shouldAdjustToUTC()));
+        break;
+      case STRING:
+        Object strVal = value.getValue(name);
+        if (strVal != null) {
+          rec.setField(name, strVal.toString());
+        }
+        break;
+      case UUID:
+        Optional.ofNullable(value.getBytes(name))
+            .ifPresent(v -> rec.setField(name, UUID.nameUUIDFromBytes(v)));
+        break;
+      case FIXED:
+        throw new UnsupportedOperationException("Fixed-precision fields are not yet supported.");
+      case BINARY:
+        Optional.ofNullable(value.getBytes(name))
+            .ifPresent(v -> rec.setField(name, ByteBuffer.wrap(v)));
+        break;
+      case DECIMAL:
+        Optional.ofNullable(value.getDecimal(name)).ifPresent(v -> rec.setField(name, v));
+        break;
+      case STRUCT:
+        Optional.ofNullable(value.getRow(name))
+            .ifPresent(
+                row ->
+                    rec.setField(
+                        name,
+                        copyRowIntoRecord(GenericRecord.create(field.type().asStructType()), row)));
+        break;
+      case LIST:
+        Iterable<@NonNull ?> icebergList = value.getIterable(name);
+        Type collectionType = ((Types.ListType) field.type()).elementType();
+
+        if (collectionType.isStructType() && icebergList != null) {
+          org.apache.iceberg.Schema innerSchema = collectionType.asStructType().asSchema();
+          ImmutableList.Builder builder = ImmutableList.builder();
+          for (Row v : (Iterable) icebergList) {
+            builder.add(beamRowToIcebergRecord(innerSchema, v));
+          }
+          icebergList = builder.build();
+        }
+        Optional.ofNullable(icebergList).ifPresent(list -> rec.setField(name, list));
+        break;
+      case MAP:
+        Map icebergMap = value.getMap(name);
+        Type valueType = ((Types.MapType) field.type()).valueType();
+        // recurse on struct types
+        if (valueType.isStructType() && icebergMap != null) {
+          org.apache.iceberg.Schema innerSchema = valueType.asStructType().asSchema();
+
+          ImmutableMap.Builder newMap = ImmutableMap.builder();
+          for (Map.Entry entry : icebergMap.entrySet()) {
+            Row row = checkStateNotNull(((Row) entry.getValue()));
+            newMap.put(checkStateNotNull(entry.getKey()), beamRowToIcebergRecord(innerSchema, row));
+          }
+          icebergMap = newMap.build();
+        }
+        Optional.ofNullable(icebergMap).ifPresent(v -> rec.setField(name, v));
+        break;
+      default:
+        // Do nothing for unsupported types
+        break;
+    }
+  }
+
+  /**
+   * Returns the appropriate value for an Iceberg timestamp field
+   *
+   * 
    If `timestamp`, we resolve incoming values to a {@link LocalDateTime}.
+   *
+   * 
    If `timestamptz`, we resolve to a UTC {@link OffsetDateTime}. Iceberg already resolves all
+   * incoming timestamps to UTC, so there is no harm in doing it from our side.
+   *
+   * 
    Valid types are:
+   *
+   * 
      
+   *   
    • {@link SqlTypes.DATETIME} --> {@link LocalDateTime}
+   *   
    • {@link Schema.FieldType.DATETIME} --> {@link Instant}
+   *   
    • {@link Schema.FieldType.INT64} --> {@link Long}
+   *   
    • {@link Schema.FieldType.STRING} --> {@link String}
+   * 
    
+   */
+  private static Object getIcebergTimestampValue(Object beamValue, boolean shouldAdjustToUtc) {
+    // timestamptz
+    if (shouldAdjustToUtc) {
+      if (beamValue instanceof java.time.Instant) { // MicrosInstant
+        OffsetDateTime epoch = java.time.Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC);
+        java.time.Instant instant = (java.time.Instant) beamValue;
+        long nanosFromEpoch =
+            TimeUnit.SECONDS.toNanos(instant.getEpochSecond()) + instant.getNano();
+        return ChronoUnit.NANOS.addTo(epoch, nanosFromEpoch);
+      } else if (beamValue instanceof LocalDateTime) { // SqlTypes.DATETIME
+        return OffsetDateTime.of((LocalDateTime) beamValue, ZoneOffset.UTC);
+      } else if (beamValue instanceof Instant) { // FieldType.DATETIME
+        return DateTimeUtil.timestamptzFromMicros(((Instant) beamValue).getMillis() * 1000L);
+      } else if (beamValue instanceof Long) { // FieldType.INT64
+        return DateTimeUtil.timestamptzFromMicros((Long) beamValue);
+      } else if (beamValue instanceof String) { // FieldType.STRING
+        return OffsetDateTime.parse((String) beamValue).withOffsetSameInstant(ZoneOffset.UTC);
+      } else {
+        throw new UnsupportedOperationException(
+            "Unsupported Beam type for Iceberg timestamp with timezone: " + beamValue.getClass());
+      }
+    }
+
+    // timestamp
+    if (beamValue instanceof java.time.Instant) { // MicrosInstant
+      java.time.Instant instant = (java.time.Instant) beamValue;
+      return DateTimeUtil.timestampFromNanos(
+          TimeUnit.SECONDS.toNanos(instant.getEpochSecond()) + instant.getNano());
+    } else if (beamValue instanceof LocalDateTime) { // SqlType.DATETIME
+      return beamValue;
+    } else if (beamValue instanceof Instant) { // FieldType.DATETIME
+      return DateTimeUtil.timestampFromMicros(((Instant) beamValue).getMillis() * 1000L);
+    } else if (beamValue instanceof Long) { // FieldType.INT64
+      return DateTimeUtil.timestampFromMicros((Long) beamValue);
+    } else if (beamValue instanceof String) { // FieldType.STRING
+      return LocalDateTime.parse((String) beamValue);
+    } else {
+      throw new UnsupportedOperationException(
+          "Unsupported Beam type for Iceberg timestamp with timezone: " + beamValue.getClass());
+    }
+  }
+
+  /** Converts an Iceberg {@link Record} to a Beam {@link Row}. */
+  public static Row icebergRecordToBeamRow(Schema schema, Record record) {
+    Row.Builder rowBuilder = Row.withSchema(schema);
+    for (Schema.Field field : schema.getFields()) {
+      boolean isNullable = field.getType().getNullable();
+      @Nullable Object icebergValue = record.getField(field.getName());
+      if (icebergValue == null) {
+        if (isNullable) {
+          rowBuilder.addValue(null);
+          continue;
+        }
+        throw new RuntimeException(
+            String.format("Received null value for required field '%s'.", field.getName()));
+      }
+      switch (field.getType().getTypeName()) {
+        case BYTE:
+        case INT16:
+        case INT32:
+        case INT64:
+        case DECIMAL: // Iceberg and Beam both use BigDecimal
+        case FLOAT: // Iceberg and Beam both use float
+        case DOUBLE: // Iceberg and Beam both use double
+        case STRING: // Iceberg and Beam both use String
+        case BOOLEAN: // Iceberg and Beam both use boolean
+          rowBuilder.addValue(icebergValue);
+          break;
+        case ARRAY:
+          checkState(
+              icebergValue instanceof List,
+              "Expected List type for field '%s' but received %s",
+              field.getName(),
+              icebergValue.getClass());
+          List<@NonNull ?> beamList = (List<@NonNull ?>) icebergValue;
+          Schema.FieldType collectionType =
+              checkStateNotNull(field.getType().getCollectionElementType());
+          // recurse on struct types
+          if (collectionType.getTypeName().isCompositeType()) {
+            Schema innerSchema = checkStateNotNull(collectionType.getRowSchema());
+            beamList =
+                beamList.stream()
+                    .map(v -> icebergRecordToBeamRow(innerSchema, (Record) v))
+                    .collect(Collectors.toList());
+          }
+          rowBuilder.addValue(beamList);
+          break;
+        case ITERABLE:
+          checkState(
+              icebergValue instanceof Iterable,
+              "Expected Iterable type for field '%s' but received %s",
+              field.getName(),
+              icebergValue.getClass());
+          Iterable<@NonNull ?> beamIterable = (Iterable<@NonNull ?>) icebergValue;
+          Schema.FieldType iterableCollectionType =
+              checkStateNotNull(field.getType().getCollectionElementType());
+          // recurse on struct types
+          if (iterableCollectionType.getTypeName().isCompositeType()) {
+            Schema innerSchema = checkStateNotNull(iterableCollectionType.getRowSchema());
+            ImmutableList.Builder builder = ImmutableList.builder();
+            for (Record v : (Iterable<@NonNull Record>) icebergValue) {
+              builder.add(icebergRecordToBeamRow(innerSchema, v));
+            }
+            beamIterable = builder.build();
+          }
+          rowBuilder.addValue(beamIterable);
+          break;
+        case MAP:
+          checkState(
+              icebergValue instanceof Map,
+              "Expected Map type for field '%s' but received %s",
+              field.getName(),
+              icebergValue.getClass());
+          Map beamMap = (Map) icebergValue;
+          Schema.FieldType valueType = checkStateNotNull(field.getType().getMapValueType());
+          // recurse on struct types
+          if (valueType.getTypeName().isCompositeType()) {
+            Schema innerSchema = checkStateNotNull(valueType.getRowSchema());
+            ImmutableMap.Builder newMap = ImmutableMap.builder();
+            for (Map.Entry entry : ((Map) icebergValue).entrySet()) {
+              Record rec = ((Record) entry.getValue());
+              newMap.put(
+                  checkStateNotNull(entry.getKey()),
+                  icebergRecordToBeamRow(innerSchema, checkStateNotNull(rec)));
+            }
+            beamMap = newMap.build();
+          }
+          rowBuilder.addValue(beamMap);
+          break;
+        case DATETIME:
+          // Iceberg uses a long for micros.
+          // Beam DATETIME uses joda's DateTime, which only supports millis,
+          // so we do lose some precision here
+          rowBuilder.addValue(getBeamDateTimeValue(icebergValue));
+          break;
+        case BYTES:
+          // Iceberg uses ByteBuffer; Beam uses byte[]
+          rowBuilder.addValue(((ByteBuffer) icebergValue).array());
+          break;
+        case ROW:
+          Record nestedRecord = (Record) icebergValue;
+          Schema nestedSchema =
+              checkArgumentNotNull(
+                  field.getType().getRowSchema(),
+                  "Corrupted schema: Row type did not have associated nested schema.");
+          rowBuilder.addValue(icebergRecordToBeamRow(nestedSchema, nestedRecord));
+          break;
+        case LOGICAL_TYPE:
+          rowBuilder.addValue(getLogicalTypeValue(icebergValue, field.getType()));
+          break;
+        default:
+          throw new UnsupportedOperationException(
+              "Unsupported Beam type: " + field.getType().getTypeName());
+      }
+    }
+    return rowBuilder.build();
+  }
+
+  private static DateTime getBeamDateTimeValue(Object icebergValue) {
+    long micros;
+    if (icebergValue instanceof OffsetDateTime) {
+      micros = DateTimeUtil.microsFromTimestamptz((OffsetDateTime) icebergValue);
+    } else if (icebergValue instanceof LocalDateTime) {
+      micros = DateTimeUtil.microsFromTimestamp((LocalDateTime) icebergValue);
+    } else if (icebergValue instanceof Long) {
+      micros = (long) icebergValue;
+    } else if (icebergValue instanceof String) {
+      return DateTime.parse((String) icebergValue);
+    } else {
+      throw new UnsupportedOperationException(
+          "Unsupported Iceberg type for Beam type DATETIME: " + icebergValue.getClass());
+    }
+    return new DateTime(micros / 1000L);
+  }
+
+  private static Object getLogicalTypeValue(Object icebergValue, Schema.FieldType type) {
+    if (icebergValue instanceof String) {
+      String strValue = (String) icebergValue;
+      if (type.isLogicalType(SqlTypes.DATE.getIdentifier())) {
+        return LocalDate.parse(strValue);
+      } else if (type.isLogicalType(SqlTypes.TIME.getIdentifier())) {
+        return LocalTime.parse(strValue);
+      } else if (type.isLogicalType(SqlTypes.DATETIME.getIdentifier())) {
+        return LocalDateTime.parse(strValue);
+      }
+    } else if (icebergValue instanceof Long) {
+      if (type.isLogicalType(SqlTypes.TIME.getIdentifier())) {
+        return DateTimeUtil.timeFromMicros((Long) icebergValue);
+      } else if (type.isLogicalType(SqlTypes.DATETIME.getIdentifier())) {
+        return DateTimeUtil.timestampFromMicros((Long) icebergValue);
+      }
+    } else if (icebergValue instanceof Integer
+        && type.isLogicalType(SqlTypes.DATE.getIdentifier())) {
+      return DateTimeUtil.dateFromDays((Integer) icebergValue);
+    } else if (icebergValue instanceof OffsetDateTime
+        && type.isLogicalType(SqlTypes.DATETIME.getIdentifier())) {
+      return ((OffsetDateTime) icebergValue)
+          .withOffsetSameInstant(ZoneOffset.UTC)
+          .toLocalDateTime();
+    }
+    // LocalDateTime, LocalDate, LocalTime
+    return icebergValue;
+  }
+
+  static  boolean isUnbounded(PCollection input) {
+    return input.isBounded().equals(PCollection.IsBounded.UNBOUNDED);
+  }
+
+  static boolean validDirectWriteLimit(@Nullable Integer directWriteByteLimit) {
+    return directWriteByteLimit != null && directWriteByteLimit >= 0;
+  }
+}
diff --git a/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java
new file mode 100644
index 000000000000..0da0d4c5968c
--- /dev/null
+++ b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/WritePartitionedRowsToFiles.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg;
+
+import static org.apache.beam.sdk.io.iceberg.AssignDestinationsAndPartitions.DESTINATION;
+import static org.apache.beam.sdk.io.iceberg.AssignDestinationsAndPartitions.PARTITION;
+import static org.apache.beam.sdk.io.iceberg.RecordWriterManager.getPartitionDataPath;
+import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.RowCoder;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionKey;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class WritePartitionedRowsToFiles
+    extends PTransform>>, PCollection> {
+  private static final Logger LOG = LoggerFactory.getLogger(WritePartitionedRowsToFiles.class);
+  private final DynamicDestinations dynamicDestinations;
+  private final IcebergCatalogConfig catalogConfig;
+  private final String filePrefix;
+
+  WritePartitionedRowsToFiles(
+      IcebergCatalogConfig catalogConfig,
+      DynamicDestinations dynamicDestinations,
+      String filePrefix) {
+    this.catalogConfig = catalogConfig;
+    this.dynamicDestinations = dynamicDestinations;
+    this.filePrefix = filePrefix;
+  }
+
+  @Override
+  public PCollection expand(PCollection>> input) {
+    Schema dataSchema =
+        ((RowCoder)
+                ((IterableCoder)
+                        ((KvCoder>) input.getCoder()).getValueCoder())
+                    .getElemCoder())
+            .getSchema();
+    return input.apply(
+        ParDo.of(new WriteDoFn(catalogConfig, dynamicDestinations, filePrefix, dataSchema)));
+  }
+
+  private static class WriteDoFn extends DoFn>, FileWriteResult> {
+
+    private final DynamicDestinations dynamicDestinations;
+    private final IcebergCatalogConfig catalogConfig;
+    private final String filePrefix;
+    private final Schema dataSchema;
+    static final Cache LAST_REFRESHED_TABLE_CACHE =
+        CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES).build();
+
+    WriteDoFn(
+        IcebergCatalogConfig catalogConfig,
+        DynamicDestinations dynamicDestinations,
+        String filePrefix,
+        Schema dataSchema) {
+      this.catalogConfig = catalogConfig;
+      this.dynamicDestinations = dynamicDestinations;
+      this.filePrefix = filePrefix;
+      this.dataSchema = dataSchema;
+    }
+
+    @ProcessElement
+    public void processElement(
+        @Element KV> element, OutputReceiver out)
+        throws Exception {
+      String tableIdentifier = checkStateNotNull(element.getKey().getString(DESTINATION));
+      String partitionPath = checkStateNotNull(element.getKey().getString(PARTITION));
+
+      IcebergDestination destination = dynamicDestinations.instantiateDestination(tableIdentifier);
+      LastRefreshedTable lastRefreshedTable = getOrCreateTable(destination, dataSchema);
+      Table table = lastRefreshedTable.table;
+      partitionPath = getPartitionDataPath(partitionPath, lastRefreshedTable.partitionFieldMap);
+
+      StructLike partitionData =
+          table.spec().isPartitioned()
+              ? DataFiles.data(table.spec(), partitionPath)
+              : new PartitionKey(table.spec(), table.schema());
+
+      String fileName =
+          destination
+              .getFileFormat()
+              .addExtension(String.format("%s-%s", filePrefix, UUID.randomUUID()));
+
+      RecordWriter writer =
+          new RecordWriter(table, destination.getFileFormat(), fileName, partitionData);
+      try {
+        Iterable sortedOrUnsortedRows =
+            IcebergRowSorter.sortRows(
+                element.getValue(), table.sortOrder(), table.schema(), dataSchema);
+        for (Row row : sortedOrUnsortedRows) {
+          Record record = IcebergUtils.beamRowToIcebergRecord(table.schema(), row);
+          writer.write(record);
+        }
+      } finally {
+        writer.close();
+      }
+
+      SerializableDataFile sdf = SerializableDataFile.from(writer.getDataFile(), partitionPath);
+      out.output(
+          FileWriteResult.builder()
+              .setTableIdentifier(destination.getTableIdentifier())
+              .setSerializableDataFile(sdf)
+              .build());
+    }
+
+    static final class LastRefreshedTable {
+      final Table table;
+      volatile Instant lastRefreshTime;
+      static final Duration STALENESS_THRESHOLD = Duration.ofMinutes(2);
+      private int specId;
+      volatile Map partitionFieldMap = Maps.newHashMap();
+
+      LastRefreshedTable(Table table, Instant lastRefreshTime) {
+        this.table = table;
+        this.specId = table.spec().specId();
+        this.lastRefreshTime = lastRefreshTime;
+        for (PartitionField partitionField : table.spec().fields()) {
+          partitionFieldMap.put(partitionField.name(), partitionField);
+        }
+      }
+
+      /**
+       * Refreshes the table metadata if it is considered stale (older than 2 minutes).
+       *
+       * 
    This method first performs a non-synchronized check on the table's freshness. This
+       * provides a lock-free fast path that avoids synchronization overhead in the common case
+       * where the table does not need to be refreshed. If the table might be stale, it then enters
+       * a synchronized block to ensure that only one thread performs the refresh operation.
+       */
+      void refreshIfStale() {
+        // Fast path: Avoid entering the synchronized block if the table is not stale.
+        if (lastRefreshTime.isAfter(Instant.now().minus(STALENESS_THRESHOLD))) {
+          return;
+        }
+        synchronized (this) {
+          if (lastRefreshTime.isBefore(Instant.now().minus(STALENESS_THRESHOLD))) {
+            table.refresh();
+            lastRefreshTime = Instant.now();
+            if (table.spec().specId() != this.specId) {
+              partitionFieldMap = Maps.newHashMap();
+              for (PartitionField partitionField : table.spec().fields()) {
+                partitionFieldMap.put(partitionField.name(), partitionField);
+              }
+              this.specId = table.spec().specId();
+            }
+          }
+        }
+      }
+    }
+
+    LastRefreshedTable getOrCreateTable(IcebergDestination destination, Schema dataSchema) {
+      TableIdentifier identifier = destination.getTableIdentifier();
+      @Nullable
+      LastRefreshedTable lastRefreshedTable = LAST_REFRESHED_TABLE_CACHE.getIfPresent(identifier);
+      if (lastRefreshedTable != null) {
+        lastRefreshedTable.refreshIfStale();
+        return lastRefreshedTable;
+      }
+
+      Namespace namespace = identifier.namespace();
+      @Nullable IcebergTableCreateConfig createConfig = destination.getTableCreateConfig();
+      PartitionSpec partitionSpec =
+          createConfig != null ? createConfig.getPartitionSpec() : PartitionSpec.unpartitioned();
+      Map tableProperties =
+          createConfig != null && createConfig.getTableProperties() != null
+              ? createConfig.getTableProperties()
+              : Maps.newHashMap();
+
+      @Nullable Table table = null;
+      synchronized (LAST_REFRESHED_TABLE_CACHE) {
+        lastRefreshedTable = LAST_REFRESHED_TABLE_CACHE.getIfPresent(identifier);
+        if (lastRefreshedTable != null) {
+          lastRefreshedTable.refreshIfStale();
+          return lastRefreshedTable;
+        }
+
+        Catalog catalog = catalogConfig.catalog();
+        // Create namespace if it does not exist yet
+        if (!namespace.isEmpty() && catalog instanceof SupportsNamespaces) {
+          SupportsNamespaces supportsNamespaces = (SupportsNamespaces) catalog;
+          if (!supportsNamespaces.namespaceExists(namespace)) {
+            try {
+              supportsNamespaces.createNamespace(namespace);
+              LOG.info("Created new namespace '{}'.", namespace);
+            } catch (AlreadyExistsException ignored) {
+              // race condition: another worker already created this namespace
+              LOG.info("Namespace `{}` already exists.", namespace);
+            }
+          }
+        }
+
+        // If table exists, just load it
+        // Note: the implementation of catalog.tableExists() will load the table to check its
+        // existence. We don't use it here to avoid double loadTable() calls.
+        try {
+          table = catalog.loadTable(identifier);
+        } catch (NoSuchTableException e) { // Otherwise, create the table
+          org.apache.iceberg.Schema tableSchema =
+              IcebergUtils.beamSchemaToIcebergSchema(dataSchema);
+          SortOrder sortOrder =
+              createConfig != null ? createConfig.getSortOrder() : SortOrder.unsorted();
+          try {
+            table =
+                catalog
+                    .buildTable(identifier, tableSchema)
+                    .withPartitionSpec(partitionSpec)
+                    .withSortOrder(sortOrder)
+                    .withProperties(tableProperties)
+                    .create();
+            LOG.info(
+                "Created Iceberg table '{}' with schema: {}\n"
+                    + ", partition spec: {}, sort order: {}, table properties: {}",
+                identifier,
+                tableSchema,
+                partitionSpec,
+                sortOrder,
+                tableProperties);
+          } catch (AlreadyExistsException ignored) {
+            // race condition: another worker already created this table
+            table = catalog.loadTable(identifier);
+          }
+        }
+      }
+      lastRefreshedTable = new LastRefreshedTable(table, Instant.now());
+      LAST_REFRESHED_TABLE_CACHE.put(identifier, lastRefreshedTable);
+      return lastRefreshedTable;
+    }
+  }
+}
diff --git a/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/test/IcebergBigQueryScaleTest.java b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/test/IcebergBigQueryScaleTest.java
new file mode 100644
index 000000000000..a479b8f08059
--- /dev/null
+++ b/scratch/iceberg-scale-test/src/main/java/org/apache/beam/sdk/io/iceberg/test/IcebergBigQueryScaleTest.java
@@ -0,0 +1,116 @@
+package org.apache.beam.sdk.io.iceberg.test;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method;
+import org.apache.beam.sdk.io.iceberg.DynamicDestinations;
+import org.apache.beam.sdk.io.iceberg.IcebergDestination;
+import org.apache.beam.sdk.io.iceberg.IcebergIO;
+import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig;
+import org.apache.beam.sdk.io.iceberg.IcebergTableCreateConfig;
+import org.apache.iceberg.DistributionMode;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.schemas.transforms.Convert;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.catalog.TableIdentifier;
+
+public class IcebergBigQueryScaleTest implements Serializable {
+
+  public static void main(String[] args) {
+    PipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().create();
+    Pipeline pipeline = Pipeline.create(options);
+
+    // 1. Define Hadoop Catalog on GCS Biglake bucket
+    Map catalogProps = new HashMap<>();
+    catalogProps.put("type", "hadoop");
+    catalogProps.put("warehouse", "gs://at-euw4-biglake-bucket/iceberg-warehouse");
+    catalogProps.put("io-impl", "org.apache.iceberg.gcp.gcs.GCSFileIO");
+
+    Map configProps = new HashMap<>();
+    configProps.put("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
+    configProps.put("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS");
+
+    IcebergCatalogConfig catalogConfig =
+        IcebergCatalogConfig.builder()
+            .setCatalogName("hadoop_catalog")
+            .setCatalogProperties(catalogProps)
+            .setConfigProperties(configProps)
+            .build();
+
+    // 2. Read crypto_ethereum.blocks partitioned table from BigQuery using High-Performance Direct Read API
+    PCollection bqRows =
+        pipeline.apply(
+            "Read Ethereum Blocks from BigQuery",
+            BigQueryIO.readTableRowsWithSchema()
+                .from("bigquery-public-data.crypto_ethereum.blocks")
+                .withMethod(Method.DIRECT_READ))
+            .apply("Convert to Beam Rows", Convert.toRows());
+
+    final Schema dataSchema = bqRows.getSchema();
+    final String salt = UUID.randomUUID().toString().substring(0, 8);
+
+    // 3. Configure dynamic destinations to create partitioned and sorted Iceberg tables
+    DynamicDestinations dynamicDestinations =
+        new DynamicDestinations() {
+          @Override
+          public Schema getDataSchema() {
+            return dataSchema;
+          }
+
+          @Override
+          public Row getData(Row element) {
+            return element;
+          }
+
+          @Override
+          public String getTableStringIdentifier(ValueInSingleWindow element) {
+            // Write all blocks to a single Ethereum blocks table
+            return "hadoop_catalog.default.ethereum_blocks_" + salt;
+          }
+
+          @Override
+          public IcebergDestination instantiateDestination(String dest) {
+            java.util.Map properties = new java.util.HashMap<>();
+            properties.put("write.format.default", "parquet");
+            properties.put("write.target-file-size-bytes", "10485760"); // 10MB target
+            properties.put("write.parquet.page-size-bytes", "1048576");  // 1MB page size
+
+            return IcebergDestination.builder()
+                .setTableIdentifier(TableIdentifier.parse(dest))
+                .setFileFormat(FileFormat.PARQUET)
+                .setTableCreateConfig(
+                    IcebergTableCreateConfig.builder()
+                        .setSchema(getDataSchema())
+                        // Partition on timestamp column (day-based) and sort by block hash
+                        .setPartitionFields(Arrays.asList("month(timestamp)"))
+                        .setSortFields(Arrays.asList("hash asc"))
+                        .setTableProperties(properties)
+                        .build())
+                .build();
+          }
+        };
+
+    bqRows.apply(
+        "Write Sorted Partitioned Blocks to GCS Iceberg Warehouse",
+        IcebergIO.writeRows(catalogConfig)
+            .to(dynamicDestinations)
+            .withDistributionMode(DistributionMode.HASH)
+            .withAutosharding());
+
+    System.out.println("Staging Dataflow pipeline graph...");
+    pipeline.run();
+    System.out.println("Dataflow pipeline launched successfully!");
+  }
+}