Skip to content

Commit 889125a

Browse files
zhjwpkuzhjwpku
committed
feat: add iceberg_data library alongside iceberg
1 parent 194db04 commit 889125a

10 files changed

Lines changed: 162 additions & 44 deletions

File tree

‎example/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ project(example)
2222

2323
set(CMAKE_CXX_STANDARD 23)
2424

25-
find_package(iceberg CONFIG REQUIRED)
25+
find_package(iceberg CONFIG REQUIRED COMPONENTS bundle rest)
2626

2727
add_executable(demo_example demo_example.cc)
2828

‎src/iceberg/CMakeLists.txt‎

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,7 @@ set(ICEBERG_INCLUDES "$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>"
2020
set(ICEBERG_SOURCES
2121
arrow_c_data_guard_internal.cc
2222
catalog/memory/in_memory_catalog.cc
23-
data/data_writer.cc
24-
data/delete_loader.cc
25-
data/equality_delete_writer.cc
26-
data/position_delete_writer.cc
27-
data/writer.cc
2823
delete_file_index.cc
29-
deletes/roaring_position_bitmap.cc
30-
deletes/position_delete_index.cc
3124
expression/aggregate.cc
3225
expression/binder.cc
3326
expression/evaluator.cc
@@ -66,9 +59,6 @@ set(ICEBERG_SOURCES
6659
partition_field.cc
6760
partition_spec.cc
6861
partition_summary.cc
69-
puffin/file_metadata.cc
70-
puffin/puffin_format.cc
71-
puffin/json_serde.cc
7262
row/arrow_array_wrapper.cc
7363
row/manifest_wrapper.cc
7464
row/partition_values.cc
@@ -134,24 +124,22 @@ list(APPEND
134124
ICEBERG_STATIC_BUILD_INTERFACE_LIBS
135125
nanoarrow::nanoarrow_static
136126
nlohmann_json::nlohmann_json
137-
roaring::roaring
138127
ZLIB::ZLIB)
139128
list(APPEND
140129
ICEBERG_SHARED_BUILD_INTERFACE_LIBS
141130
nanoarrow::nanoarrow_shared
142131
nlohmann_json::nlohmann_json
143-
roaring::roaring
144132
ZLIB::ZLIB)
145133
list(APPEND
146134
ICEBERG_STATIC_INSTALL_INTERFACE_LIBS
147135
"$<IF:$<BOOL:${NANOARROW_VENDORED}>,iceberg::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_static>,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>"
148136
"$<IF:$<BOOL:${NLOHMANN_JSON_VENDORED}>,iceberg::nlohmann_json,$<IF:$<TARGET_EXISTS:nlohmann_json::nlohmann_json>,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>"
149-
"$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
137+
)
150138
list(APPEND
151139
ICEBERG_SHARED_INSTALL_INTERFACE_LIBS
152140
"$<IF:$<BOOL:${NANOARROW_VENDORED}>,iceberg::nanoarrow_shared,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_shared>,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>"
153141
"$<IF:$<BOOL:${NLOHMANN_JSON_VENDORED}>,iceberg::nlohmann_json,$<IF:$<TARGET_EXISTS:nlohmann_json::nlohmann_json>,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>"
154-
"$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
142+
)
155143

156144
add_iceberg_lib(iceberg
157145
SOURCES
@@ -169,6 +157,60 @@ add_iceberg_lib(iceberg
169157
OUTPUTS
170158
ICEBERG_LIBRARIES)
171159

160+
set(ICEBERG_DATA_SOURCES
161+
data/data_writer.cc
162+
data/delete_loader.cc
163+
data/equality_delete_writer.cc
164+
data/position_delete_writer.cc
165+
data/writer.cc
166+
deletes/position_delete_index.cc
167+
deletes/roaring_position_bitmap.cc
168+
puffin/file_metadata.cc
169+
puffin/json_serde.cc
170+
puffin/puffin_format.cc)
171+
172+
set(ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS)
173+
set(ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS)
174+
set(ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS)
175+
set(ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS)
176+
177+
list(APPEND ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS
178+
"$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
179+
roaring::roaring)
180+
list(APPEND ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS
181+
"$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
182+
roaring::roaring)
183+
list(APPEND
184+
ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS
185+
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_static>,iceberg::iceberg_static,iceberg::iceberg_shared>"
186+
"$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
187+
list(APPEND
188+
ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS
189+
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_shared>,iceberg::iceberg_shared,iceberg::iceberg_static>"
190+
"$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
191+
192+
add_iceberg_lib(iceberg_data
193+
SOURCES
194+
${ICEBERG_DATA_SOURCES}
195+
EXTRA_INCLUDES
196+
${ICEBERG_INCLUDES}
197+
SHARED_LINK_LIBS
198+
${ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS}
199+
STATIC_LINK_LIBS
200+
${ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS}
201+
STATIC_INSTALL_INTERFACE_LIBS
202+
${ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS}
203+
SHARED_INSTALL_INTERFACE_LIBS
204+
${ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS})
205+
206+
if(TARGET iceberg_data_shared)
207+
target_compile_definitions(iceberg_data_shared PRIVATE ICEBERG_EXPORTING)
208+
endif()
209+
210+
if(TARGET iceberg_data_static)
211+
target_compile_definitions(iceberg_data_static PRIVATE ICEBERG_STATIC)
212+
endif()
213+
172214
iceberg_install_all_headers(iceberg)
173215

174216
add_subdirectory(catalog)
@@ -210,29 +252,29 @@ if(ICEBERG_BUILD_BUNDLE)
210252

211253
list(APPEND
212254
ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
213-
"$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
255+
"$<IF:$<TARGET_EXISTS:iceberg_data_static>,iceberg_data_static,iceberg_data_shared>"
214256
"$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>"
215257
"$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>"
216258
"$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_static>,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>"
217259
)
218260
list(APPEND
219261
ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
220-
"$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
262+
"$<IF:$<TARGET_EXISTS:iceberg_data_shared>,iceberg_data_shared,iceberg_data_static>"
221263
"$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>"
222264
"$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>"
223265
"$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>"
224266
)
225267

226268
list(APPEND
227269
ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
228-
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_static>,iceberg::iceberg_static,iceberg::iceberg_shared>"
270+
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_data_static>,iceberg::iceberg_data_static,iceberg::iceberg_data_shared>"
229271
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>>"
230272
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>>"
231273
"$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_static>,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>>"
232274
)
233275
list(APPEND
234276
ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS
235-
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_shared>,iceberg::iceberg_shared,iceberg::iceberg_static>"
277+
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_data_shared>,iceberg::iceberg_data_shared,iceberg::iceberg_data_static>"
236278
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>>"
237279
"$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>>"
238280
"$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>"

‎src/iceberg/arrow_c_data_guard_internal.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,11 @@
2222
#include <nanoarrow/nanoarrow.h>
2323

2424
#include "iceberg/arrow_c_data.h"
25+
#include "iceberg/iceberg_export.h"
2526

2627
namespace iceberg::internal {
2728

28-
class ArrowArrayGuard {
29+
class ICEBERG_EXPORT ArrowArrayGuard {
2930
public:
3031
explicit ArrowArrayGuard(ArrowArray* array) : array_(array) {}
3132
~ArrowArrayGuard();
@@ -34,7 +35,7 @@ class ArrowArrayGuard {
3435
ArrowArray* array_;
3536
};
3637

37-
class ArrowSchemaGuard {
38+
class ICEBERG_EXPORT ArrowSchemaGuard {
3839
public:
3940
explicit ArrowSchemaGuard(ArrowSchema* schema) : schema_(schema) {}
4041
~ArrowSchemaGuard();
@@ -43,7 +44,7 @@ class ArrowSchemaGuard {
4344
ArrowSchema* schema_;
4445
};
4546

46-
class ArrowArrayViewGuard {
47+
class ICEBERG_EXPORT ArrowArrayViewGuard {
4748
public:
4849
explicit ArrowArrayViewGuard(ArrowArrayView* view) : view_(view) {}
4950
~ArrowArrayViewGuard();
@@ -52,7 +53,7 @@ class ArrowArrayViewGuard {
5253
ArrowArrayView* view_;
5354
};
5455

55-
class ArrowArrayBufferGuard {
56+
class ICEBERG_EXPORT ArrowArrayBufferGuard {
5657
public:
5758
explicit ArrowArrayBufferGuard(ArrowBuffer* buffer) : buffer_(buffer) {}
5859
~ArrowArrayBufferGuard();

‎src/iceberg/data/meson.build‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
install_headers(
19+
[
20+
'data_writer.h',
21+
'delete_loader.h',
22+
'equality_delete_writer.h',
23+
'position_delete_writer.h',
24+
'writer.h',
25+
],
26+
subdir: 'iceberg/data',
27+
)

‎src/iceberg/file_writer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636

3737
namespace iceberg {
3838

39-
class WriterProperties : public ConfigBase<WriterProperties> {
39+
class ICEBERG_EXPORT WriterProperties : public ConfigBase<WriterProperties> {
4040
public:
4141
template <typename T>
4242
using Entry = const ConfigBase<WriterProperties>::Entry<T>;

‎src/iceberg/iceberg-config.cmake.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
#
2525
# iceberg::iceberg_shared
2626
# iceberg::iceberg_static
27+
# iceberg::iceberg_data_shared
28+
# iceberg::iceberg_data_static
2729
# iceberg::iceberg_bundle_shared
2830
# iceberg::iceberg_bundle_static
2931
# iceberg::iceberg_rest_shared

‎src/iceberg/iceberg_export.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,5 @@
5353

5454
# define ICEBERG_TEMPLATE_EXPORT
5555
# define ICEBERG_TEMPLATE_CLASS_EXPORT ICEBERG_EXPORT
56-
# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_TEMPLATE_EXPORT
56+
# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_TEMPLATE_CLASS_EXPORT
5757
#endif

‎src/iceberg/meson.build‎

Lines changed: 42 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,7 @@ iceberg_include_dir = include_directories('..')
4242
iceberg_sources = files(
4343
'arrow_c_data_guard_internal.cc',
4444
'catalog/memory/in_memory_catalog.cc',
45-
'data/data_writer.cc',
46-
'data/delete_loader.cc',
47-
'data/equality_delete_writer.cc',
48-
'data/position_delete_writer.cc',
49-
'data/writer.cc',
5045
'delete_file_index.cc',
51-
'deletes/position_delete_index.cc',
52-
'deletes/roaring_position_bitmap.cc',
5346
'expression/aggregate.cc',
5447
'expression/binder.cc',
5548
'expression/evaluator.cc',
@@ -88,9 +81,6 @@ iceberg_sources = files(
8881
'partition_field.cc',
8982
'partition_spec.cc',
9083
'partition_summary.cc',
91-
'puffin/file_metadata.cc',
92-
'puffin/json_serde.cc',
93-
'puffin/puffin_format.cc',
9484
'row/arrow_array_wrapper.cc',
9585
'row/manifest_wrapper.cc',
9686
'row/partition_values.cc',
@@ -148,6 +138,19 @@ iceberg_sources = files(
148138
'util/uuid.cc',
149139
)
150140

141+
iceberg_data_sources = files(
142+
'data/data_writer.cc',
143+
'data/delete_loader.cc',
144+
'data/equality_delete_writer.cc',
145+
'data/position_delete_writer.cc',
146+
'data/writer.cc',
147+
'deletes/position_delete_index.cc',
148+
'deletes/roaring_position_bitmap.cc',
149+
'puffin/file_metadata.cc',
150+
'puffin/json_serde.cc',
151+
'puffin/puffin_format.cc',
152+
)
153+
151154
# CRoaring does not export symbols, so on Windows it must
152155
# be used as a static lib
153156
croaring_needs_static = (
@@ -159,7 +162,7 @@ nanoarrow_dep = dependency('nanoarrow')
159162
nlohmann_json_dep = dependency('nlohmann_json')
160163
zlib_dep = dependency('zlib')
161164

162-
iceberg_deps = [croaring_dep, nanoarrow_dep, nlohmann_json_dep, zlib_dep]
165+
iceberg_deps = [nanoarrow_dep, nlohmann_json_dep, zlib_dep]
163166

164167
iceberg_lib = library(
165168
'iceberg',
@@ -184,8 +187,35 @@ iceberg_dep = declare_dependency(
184187
compile_args: iceberg_interface_args,
185188
)
186189
meson.override_dependency('iceberg', iceberg_dep)
190+
191+
iceberg_data_deps = [iceberg_dep, croaring_dep]
192+
iceberg_data_lib = library(
193+
'iceberg_data',
194+
sources: iceberg_data_sources,
195+
dependencies: iceberg_data_deps,
196+
include_directories: iceberg_include_dir,
197+
install: true,
198+
gnu_symbol_visibility: 'inlineshidden',
199+
cpp_shared_args: ['-DICEBERG_EXPORTING'],
200+
cpp_static_args: ['-DICEBERG_STATIC'],
201+
)
202+
203+
iceberg_data_interface_args = []
204+
if get_option('default_library') != 'shared'
205+
iceberg_data_interface_args += ['-DICEBERG_STATIC']
206+
endif
207+
208+
iceberg_data_dep = declare_dependency(
209+
link_with: iceberg_data_lib,
210+
dependencies: iceberg_data_deps,
211+
include_directories: iceberg_include_dir,
212+
compile_args: iceberg_data_interface_args,
213+
)
214+
meson.override_dependency('iceberg-data', iceberg_data_dep)
215+
187216
pkg = import('pkgconfig')
188217
pkg.generate(iceberg_lib)
218+
pkg.generate(iceberg_data_lib)
189219

190220
install_headers(
191221
[
@@ -233,6 +263,7 @@ install_headers(
233263
)
234264

235265
subdir('catalog')
266+
subdir('data')
236267
subdir('deletes')
237268
subdir('expression')
238269
subdir('manifest')

‎src/iceberg/test/CMakeLists.txt‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources")
3131
configure_file("test_config.h.in" "test_config.h")
3232

3333
function(add_iceberg_test test_name)
34-
set(options USE_BUNDLE)
34+
set(options USE_BUNDLE USE_DATA)
3535
set(oneValueArgs)
3636
set(multiValueArgs SOURCES)
3737
cmake_parse_arguments(ARG
@@ -47,6 +47,10 @@ function(add_iceberg_test test_name)
4747

4848
if(ARG_USE_BUNDLE)
4949
target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gmock_main)
50+
elseif(ARG_USE_DATA)
51+
target_link_libraries(${test_name}
52+
PRIVATE "$<IF:$<TARGET_EXISTS:iceberg_data_static>,iceberg_data_static,iceberg_data_shared>"
53+
GTest::gmock_main)
5054
else()
5155
target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gmock_main)
5256
endif()
@@ -109,6 +113,7 @@ add_iceberg_test(json_serde_test
109113
schema_json_test.cc)
110114

111115
add_iceberg_test(util_test
116+
USE_DATA
112117
SOURCES
113118
bucket_util_test.cc
114119
config_test.cc
@@ -128,9 +133,13 @@ add_iceberg_test(util_test
128133
uuid_test.cc
129134
visit_type_test.cc)
130135

131-
add_iceberg_test(roaring_test SOURCES roaring_test.cc)
136+
add_iceberg_test(roaring_test USE_DATA SOURCES roaring_test.cc)
132137

133-
add_iceberg_test(puffin_test SOURCES puffin_format_test.cc puffin_json_test.cc)
138+
add_iceberg_test(puffin_test
139+
USE_DATA
140+
SOURCES
141+
puffin_format_test.cc
142+
puffin_json_test.cc)
134143

135144
if(ICEBERG_BUILD_BUNDLE)
136145
add_iceberg_test(avro_test

0 commit comments

Comments
 (0)