Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
11 changes: 5 additions & 6 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ maven.install(
"io.opencensus:opencensus-api:0.31.0",
"io.opencensus:opencensus-contrib-grpc-metrics:0.31.0",
"junit:junit:4.13.2",
"net.java.dev.javacc:javacc:7.0.3",
"net.java.dev.javacc:javacc:7.0.13",
],
known_contributing_modules = [
"cloud_spanner_emulator",
Expand Down Expand Up @@ -228,17 +228,16 @@ http_archive(
)

bazel_dep(name = "google_benchmark", version = "1.8.3", repo_name = "com_github_google_benchmark")
bazel_dep(name = "googlesql", version = "2026.1.1", repo_name = "com_google_googlesql")
bazel_dep(name = "googlesql", version = "2026.7.2", repo_name = "com_google_googlesql")
archive_override(
module_name = "googlesql",
# Patches applied:
# - Give visibility to GoogleSQL's base library to reuse some utilities
# - Correct the module version to semantic versioning format (2026.01.1 -> 2026.1.1)
integrity = "sha256-ZnGi8IIn5G5QnV4sjP57k3ZVvSzmaPWcLSsMtvrWUTg=",
integrity = "sha256-9zT65nfWcsYaJ41WJHO/iuLQF/R9TdKzf6Jh5RNoUng=",
patch_strip = 1,
patches = ["//build/bazel:googlesql.patch"],
strip_prefix = "googlesql-2026.01.1",
urls = ["https://github.com/google/googlesql/archive/refs/tags/2026.01.1.zip"],
strip_prefix = "googlesql-2026.7.2",
urls = ["https://github.com/google/googlesql/archive/refs/tags/2026.7.2.zip"],
)

################################################################################
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ Notable supported features:

- [GQL support](https://docs.cloud.google.com/spanner/docs/reference/standard-sql/graph-intro)

- [Graph Algorithms](https://cloud.google.com/spanner/docs/graph/graph-algorithms-overview) (Graph Algorithms are supported in the emulator for parsing and query validation, but they will always return an empty result set)

- DML sequence numbers

- Non-SQL read and write methods
Expand Down
1 change: 1 addition & 0 deletions backend/actions/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ cc_test(
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/time",
"@com_google_absl//absl/types:span",
"@com_google_absl//absl/types:variant",
"@com_google_googlesql//googlesql/base/testing:status_matchers",
Expand Down
4 changes: 2 additions & 2 deletions backend/actions/change_stream.cc
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@ void CloudValueToJSONValue(const googlesql::Value value, JSONValueRef& ref) {
break;
}
case googlesql::TYPE_TIMESTAMP: {
ref.SetString(absl::FormatTime("%Y-%m-%d%ET%H:%M:%SZ", value.ToTime(),
ref.SetString(absl::FormatTime("%Y-%m-%d%ET%H:%M:%E*SZ", value.ToTime(),
absl::UTCTimeZone()));
break;
}
Expand Down Expand Up @@ -652,7 +652,7 @@ absl::StatusOr<WriteOp> ConvertDataChangeRecordToWriteOp(
GOOGLESQL_ASSIGN_OR_RETURN(Key change_stream_data_table_key,
ComputeChangeStreamDataTableKey(
record.partition_token, record.commit_timestamp,
record.server_transaction_id, record.record_sequence,
record.record_sequence, record.server_transaction_id,
change_stream->change_stream_data_table()->Name()));
std::vector<googlesql::Value> values;
values.push_back(record.partition_token);
Expand Down
66 changes: 66 additions & 0 deletions backend/actions/change_stream_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include "absl/log/log.h"
#include "absl/strings/escaping.h"
#include "absl/strings/str_cat.h"
#include "absl/time/time.h"
#include "absl/types/span.h"
#include "absl/types/variant.h"
#include "backend/actions/ops.h"
Expand Down Expand Up @@ -1384,6 +1385,71 @@ TEST_F(ChangeStreamTest, ProtoValueAndTypes) {
"\"],\"proto_col\":\"", encoded_proto, "\"}"))));
}

TEST_F(ChangeStreamTest, TimestampValueAndTypes) {
GOOGLESQL_ASSERT_OK_AND_ASSIGN(auto t_schema, emulator::test::CreateSchemaFromDDL(
{
R"(
CREATE TABLE TimestampTable (
commit_ts TIMESTAMP NOT NULL,
k INT64
) PRIMARY KEY (commit_ts, k)
)",
R"(
CREATE CHANGE STREAM ChangeStream_TimestampTable FOR TimestampTable OPTIONS ( value_capture_type = 'NEW_VALUES' )
)"},
&type_factory_));

const Table* t_table = t_schema->FindTable("TimestampTable");
const ChangeStream* t_stream =
t_schema->FindChangeStream("ChangeStream_TimestampTable");

set_up_partition_token_for_change_stream_partition_table(t_stream, store());

// Insert base table entries with fractional timestamp.
absl::Time time1;
std::string err;
ASSERT_TRUE(absl::ParseTime("%Y-%m-%dT%H:%M:%E*SZ",
"1970-01-21T14:09:51.123456789Z", &time1, &err))
<< err;
googlesql::Value ts_val1 = googlesql::Value::Timestamp(time1);

absl::Time time2;
ASSERT_TRUE(absl::ParseTime("%Y-%m-%dT%H:%M:%E*SZ",
"1970-01-21T14:09:51.123000000Z", &time2, &err))
<< err;
googlesql::Value ts_val2 = googlesql::Value::Timestamp(time2);

std::vector<WriteOp> buffered_write_ops;
buffered_write_ops.push_back(Insert(t_table, Key({ts_val1, Int64(42)}),
t_table->columns(),
{ts_val1, Int64(42)}));
buffered_write_ops.push_back(Insert(t_table, Key({ts_val2, Int64(43)}),
t_table->columns(),
{ts_val2, Int64(43)}));
GOOGLESQL_ASSERT_OK_AND_ASSIGN(
std::vector<WriteOp> change_stream_write_ops,
BuildChangeStreamWriteOps(t_schema.get(), buffered_write_ops, store(), 1,
/*exclude_txn_from_change_streams=*/false));

// Verify change stream entry is added to the transaction buffer.
ASSERT_EQ(change_stream_write_ops.size(), 1);
WriteOp op = change_stream_write_ops[0];
auto* operation = std::get_if<InsertOp>(&op);
ASSERT_NE(operation, nullptr);

// Verify elements in the primary key show correct formatted timestamp
// preserving fractional digits and trailing Z.
googlesql::Value mod_keys = operation->values[10];
ASSERT_EQ(mod_keys.num_elements(), 2);
ASSERT_EQ(
mod_keys.element(0),
googlesql::Value(String(
"{\"commit_ts\":\"1970-01-21T14:09:51.123456789Z\",\"k\":\"42\"}")));
ASSERT_EQ(mod_keys.element(1),
googlesql::Value(String(
"{\"commit_ts\":\"1970-01-21T14:09:51.123Z\",\"k\":\"43\"}")));
}

} // namespace
} // namespace backend
} // namespace emulator
Expand Down
2 changes: 2 additions & 0 deletions backend/database/change_stream/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ cc_test(
"//backend/schema/updater:schema_updater",
"//backend/transaction:read_only_transaction",
"//common:clock",
"//common:feature_flags",
"//tests/common:proto_matchers",
"//tests/common:scoped_feature_flags_setter",
"@com_github_google_benchmark//:benchmark",
"@com_github_grpc_grpc//:grpc++",
"@com_google_absl//absl/flags:flag",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,11 @@ absl::Status ChangeStreamPartitionChurner::ChurnPartitions(
for (const auto& [churn_type, partition_tokens] : churned_partitions) {
// Churn the tokens retrieved above.
if (churn_type == "MOVE") {
// Skip MOVE churning for mutable key range change streams.
if (change_stream->partition_mode() ==
kChangeStreamPartitionModeMutableKeyRange) {
continue;
}
// Make sure to move each partition.
for (const auto& partition_token : partition_tokens) {
GOOGLESQL_RETURN_IF_ERROR(
Expand Down
174 changes: 174 additions & 0 deletions backend/database/change_stream/change_stream_partition_churner_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
#include "backend/transaction/options.h"
#include "backend/transaction/read_only_transaction.h"
#include "common/clock.h"
#include "common/feature_flags.h"
#include "tests/common/scoped_feature_flags_setter.h"
#include "googlesql/base/status_macros.h"

namespace google {
Expand Down Expand Up @@ -530,6 +532,178 @@ TEST_F(ChangeStreamPartitionChurnerTest, ChangeStreamSplitAndMerge) {
}
}

TEST_F(ChangeStreamPartitionChurnerTest,
MutableKeyRangeChangeStreamNoMoveChurn) {
EmulatorFeatureFlags::Flags flags;
flags.enable_mutable_key_range_change_stream = true;
test::ScopedEmulatorFeatureFlagsSetter setter(flags);
absl::SetFlag(&FLAGS_enable_change_stream_churning, false);

std::string change_stream = "mutable_key_range_change_stream";
std::string create_change_stream =
"CREATE CHANGE STREAM " + change_stream +
" OPTIONS (partition_mode = 'MUTABLE_KEY_RANGE')";
std::vector<std::string> update_statements = {create_change_stream};

absl::Status backfill_status;
int completed_statements;
absl::Time commit_ts;
GOOGLESQL_ASSERT_OK(
db_->UpdateSchema(SchemaChangeOperation{.statements = update_statements},
&completed_statements, &commit_ts, &backfill_status));

// Verify that 2 active partitions are created initially (one SPLIT, one
// MOVE).
GOOGLESQL_ASSERT_OK_AND_ASSIGN(StaleAndActivePartitions stale_and_active_partitions,
GetChangeStreamPartitions(change_stream, db_.get()));
ASSERT_EQ(stale_and_active_partitions.stale_partitions.size(), 0);
ASSERT_EQ(stale_and_active_partitions.active_partitions.size(), 2);

std::string split_token;
std::string move_token;
for (const auto& partition : stale_and_active_partitions.active_partitions) {
if (partition.next_churn == "SPLIT") {
split_token = partition.partition_token;
} else if (partition.next_churn == "MOVE") {
move_token = partition.partition_token;
}
}
ASSERT_FALSE(split_token.empty());
ASSERT_FALSE(move_token.empty());

// Sleep to let churning happen.
absl::SleepFor(
absl::GetFlag(FLAGS_change_stream_churn_thread_sleep_interval) * 5);
ChurnPartitionsForChangeStream(change_stream);

// Verify that:
// - The SPLIT partition did churn (it is now stale).
// - The MOVE partition did NOT churn (still active, no children, end time is
// null).
GOOGLESQL_ASSERT_OK_AND_ASSIGN(stale_and_active_partitions,
GetChangeStreamPartitions(change_stream, db_.get()));

// The MOVE partition should still be in active_partitions, and have no
// children.
bool found_move_partition = false;
for (const auto& partition : stale_and_active_partitions.active_partitions) {
if (partition.partition_token == move_token) {
found_move_partition = true;
EXPECT_TRUE(partition.children.empty());
EXPECT_EQ(partition.next_churn, "MOVE");
}
}
EXPECT_TRUE(found_move_partition);

// The MOVE partition should NOT be in stale_partitions.
for (const auto& partition : stale_and_active_partitions.stale_partitions) {
EXPECT_NE(partition.partition_token, move_token);
}

// The SPLIT partition should now be in stale_partitions.
bool found_split_partition_in_stale = false;
for (const auto& partition : stale_and_active_partitions.stale_partitions) {
if (partition.partition_token == split_token) {
found_split_partition_in_stale = true;
}
}
EXPECT_TRUE(found_split_partition_in_stale);

// Find the children of the split partition.
std::vector<std::string> split_children;
for (const auto& partition : stale_and_active_partitions.stale_partitions) {
if (partition.partition_token == split_token) {
split_children = partition.children;
}
}
ASSERT_EQ(split_children.size(), 2);

// Verify the children are active and have next_churn == "MERGE".
std::string merge_token_one = split_children[0];
std::string merge_token_two = split_children[1];
bool found_merge_one = false;
bool found_merge_two = false;
std::string active_tokens_str = "";
for (const auto& partition : stale_and_active_partitions.active_partitions) {
active_tokens_str += "\n " + partition.partition_token +
" (next_churn: " + partition.next_churn + ")";
if (partition.partition_token == merge_token_one) {
found_merge_one = true;
EXPECT_EQ(partition.next_churn, "MERGE");
}
if (partition.partition_token == merge_token_two) {
found_merge_two = true;
EXPECT_EQ(partition.next_churn, "MERGE");
}
}
EXPECT_TRUE(found_merge_one)
<< "Looking for merge_token_one: " << merge_token_one
<< "\nActive partitions:" << active_tokens_str;
EXPECT_TRUE(found_merge_two)
<< "Looking for merge_token_two: " << merge_token_two
<< "\nActive partitions:" << active_tokens_str;

// Churn again to merge.
absl::SleepFor(
absl::GetFlag(FLAGS_change_stream_churn_thread_sleep_interval) * 5);
ChurnPartitionsForChangeStream(change_stream);

GOOGLESQL_ASSERT_OK_AND_ASSIGN(stale_and_active_partitions,
GetChangeStreamPartitions(change_stream, db_.get()));

// Verify the MERGE partitions are now stale.
found_merge_one = false;
found_merge_two = false;
for (const auto& partition : stale_and_active_partitions.stale_partitions) {
if (partition.partition_token == merge_token_one) {
found_merge_one = true;
}
if (partition.partition_token == merge_token_two) {
found_merge_two = true;
}
}
EXPECT_TRUE(found_merge_one);
EXPECT_TRUE(found_merge_two);

// Verify they merged into a new active SPLIT partition.
// The stale MERGE partitions should both have the same child.
std::string merged_child_one;
std::string merged_child_two;
for (const auto& partition : stale_and_active_partitions.stale_partitions) {
if (partition.partition_token == merge_token_one) {
ASSERT_EQ(partition.children.size(), 1);
merged_child_one = partition.children[0];
}
if (partition.partition_token == merge_token_two) {
ASSERT_EQ(partition.children.size(), 1);
merged_child_two = partition.children[0];
}
}
ASSERT_EQ(merged_child_one, merged_child_two);
ASSERT_FALSE(merged_child_one.empty());

// Verify the merged child is active and has next_churn == "SPLIT".
bool found_merged_child = false;
for (const auto& partition : stale_and_active_partitions.active_partitions) {
if (partition.partition_token == merged_child_one) {
found_merged_child = true;
EXPECT_EQ(partition.next_churn, "SPLIT");
}
}
EXPECT_TRUE(found_merged_child);

// Verify the MOVE partition is STILL active and hasn't churned.
found_move_partition = false;
for (const auto& partition : stale_and_active_partitions.active_partitions) {
if (partition.partition_token == move_token) {
found_move_partition = true;
EXPECT_TRUE(partition.children.empty());
EXPECT_EQ(partition.next_churn, "MOVE");
}
}
EXPECT_TRUE(found_move_partition);
}

} // namespace backend
} // namespace emulator
} // namespace spanner
Expand Down
Loading
Loading