diff --git a/accessors/clients/dataflow/dataflow_client.go b/accessors/clients/dataflow/dataflow_client.go new file mode 100644 index 0000000000..94ac4584f7 --- /dev/null +++ b/accessors/clients/dataflow/dataflow_client.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dataflowclient + +import ( + "context" + "fmt" + "sync" + + dataflow "cloud.google.com/go/dataflow/apiv1beta3" +) + +var once sync.Once +var dfClient *dataflow.FlexTemplatesClient + +// This function is declared as a global variable to make it testable. The unit +// tests edit this function, acting like a double. +var newFlexTemplatesClient = dataflow.NewFlexTemplatesClient + +func GetOrCreateClient(ctx context.Context) (*dataflow.FlexTemplatesClient, error) { + var err error + if dfClient == nil { + once.Do(func() { + dfClient, err = newFlexTemplatesClient(ctx) + }) + if err != nil { + return nil, fmt.Errorf("failed to create dataflow client: %v", err) + } + return dfClient, nil + } + return dfClient, nil +} diff --git a/accessors/clients/dataflow/dataflow_client_test.go b/accessors/clients/dataflow/dataflow_client_test.go new file mode 100644 index 0000000000..a76cc98fc4 --- /dev/null +++ b/accessors/clients/dataflow/dataflow_client_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dataflowclient + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + + dataflow "cloud.google.com/go/dataflow/apiv1beta3" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "google.golang.org/api/option" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func resetTest() { + dfClient = nil + once = sync.Once{} +} + +func TestGetOrCreateClient_Basic(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newFlexTemplatesClient + defer func() { newFlexTemplatesClient = oldFunc }() + newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) { + return &dataflow.FlexTemplatesClient{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newFlexTemplatesClient + defer func() { newFlexTemplatesClient = oldFunc }() + + newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) { + return &dataflow.FlexTemplatesClient{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) + // Explicitly set the client to nil. Running GetOrCreateClient should not create a + // new client since sync would already be executed. + dfClient = nil + newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) { + return nil, fmt.Errorf("test error") + } + c, err = GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newFlexTemplatesClient + defer func() { newFlexTemplatesClient = oldFunc }() + + newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) { + return &dataflow.FlexTemplatesClient{}, nil + } + oldC, err := GetOrCreateClient(ctx) + assert.NotNil(t, oldC) + assert.Nil(t, err) + + // Explicitly reset once. Running GetOrCreateClient should not create a + // new client the if condition should prevent it. + once = sync.Once{} + newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) { + return nil, fmt.Errorf("test error") + } + newC, err := GetOrCreateClient(ctx) + assert.Equal(t, oldC, newC) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_Error(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newFlexTemplatesClient + defer func() { newFlexTemplatesClient = oldFunc }() + + newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) { + return nil, fmt.Errorf("test error") + } + c, err := GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.NotNil(t, err) +} diff --git a/accessors/clients/spanner/admin/admin_client.go b/accessors/clients/spanner/admin/admin_client.go new file mode 100644 index 0000000000..220621ee09 --- /dev/null +++ b/accessors/clients/spanner/admin/admin_client.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spanneradmin + +import ( + "context" + "fmt" + "sync" + + database "cloud.google.com/go/spanner/admin/database/apiv1" +) + +var once sync.Once +var spannerAdminClient *database.DatabaseAdminClient + +// This function is declared as a global variable to make it testable. The unit +// tests edit this function, acting like a double. +var newDatabaseAdminClient = database.NewDatabaseAdminClient + +func GetOrCreateClient(ctx context.Context) (*database.DatabaseAdminClient, error) { + var err error + if spannerAdminClient == nil { + once.Do(func() { + spannerAdminClient, err = newDatabaseAdminClient(ctx) + }) + if err != nil { + return nil, fmt.Errorf("failed to create spanner admin client: %v", err) + } + return spannerAdminClient, nil + } + return spannerAdminClient, nil +} diff --git a/accessors/clients/spanner/admin/admin_client_test.go b/accessors/clients/spanner/admin/admin_client_test.go new file mode 100644 index 0000000000..7c911ee096 --- /dev/null +++ b/accessors/clients/spanner/admin/admin_client_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spanneradmin + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + + database "cloud.google.com/go/spanner/admin/database/apiv1" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "google.golang.org/api/option" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func resetTest() { + spannerAdminClient = nil + once = sync.Once{} +} + +func TestGetOrCreateClient_Basic(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newDatabaseAdminClient + defer func() { newDatabaseAdminClient = oldFunc }() + newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) { + return &database.DatabaseAdminClient{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newDatabaseAdminClient + defer func() { newDatabaseAdminClient = oldFunc }() + + newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) { + return &database.DatabaseAdminClient{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) + // Explicitly set the client to nil. Running GetOrCreateClient should not create a + // new client since sync would already be executed. + spannerAdminClient = nil + newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) { + return nil, fmt.Errorf("test error") + } + c, err = GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newDatabaseAdminClient + defer func() { newDatabaseAdminClient = oldFunc }() + + newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) { + return &database.DatabaseAdminClient{}, nil + } + oldC, err := GetOrCreateClient(ctx) + assert.NotNil(t, oldC) + assert.Nil(t, err) + + // Explicitly reset once. Running GetOrCreateClient should not create a + // new client the if condition should prevent it. + once = sync.Once{} + newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) { + return nil, fmt.Errorf("test error") + } + newC, err := GetOrCreateClient(ctx) + assert.Equal(t, oldC, newC) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_Error(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newDatabaseAdminClient + defer func() { newDatabaseAdminClient = oldFunc }() + + newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) { + return nil, fmt.Errorf("test error") + } + c, err := GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.NotNil(t, err) +} diff --git a/accessors/clients/spanner/client/spanner_client.go b/accessors/clients/spanner/client/spanner_client.go new file mode 100644 index 0000000000..d6edd81ece --- /dev/null +++ b/accessors/clients/spanner/client/spanner_client.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spannerclient + +import ( + "context" + "fmt" + "sync" + + sp "cloud.google.com/go/spanner" +) + +var once sync.Once +var spannerClient *sp.Client + +// This function is declared as a global variable to make it testable. The unit +// tests edit this function, acting like a double. +var newClient = sp.NewClient + +func GetOrCreateClient(ctx context.Context, dbURI string) (*sp.Client, error) { + var err error + if spannerClient == nil { + once.Do(func() { + spannerClient, err = newClient(ctx, dbURI) + }) + if err != nil { + return nil, fmt.Errorf("failed to create spanner database client: %v", err) + } + return spannerClient, nil + } + return spannerClient, nil +} diff --git a/accessors/clients/spanner/client/spanner_client_test.go b/accessors/clients/spanner/client/spanner_client_test.go new file mode 100644 index 0000000000..66f5059591 --- /dev/null +++ b/accessors/clients/spanner/client/spanner_client_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spannerclient + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + + sp "cloud.google.com/go/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "google.golang.org/api/option" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func resetTest() { + spannerClient = nil + once = sync.Once{} +} + +func TestGetOrCreateClient_Basic(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return &sp.Client{}, nil + } + c, err := GetOrCreateClient(ctx, "testURI") + assert.NotNil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return &sp.Client{}, nil + } + c, err := GetOrCreateClient(ctx, "testURI") + assert.NotNil(t, c) + assert.Nil(t, err) + // Explicitly set the client to nil. Running GetOrCreateClient should not create a + // new client since sync would already be executed. + spannerClient = nil + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return nil, fmt.Errorf("test error") + } + c, err = GetOrCreateClient(ctx, "testURI") + assert.Nil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return &sp.Client{}, nil + } + oldC, err := GetOrCreateClient(ctx, "testURI") + assert.NotNil(t, oldC) + assert.Nil(t, err) + + // Explicitly reset once. Running GetOrCreateClient should not create a + // new client the if condition should prevent it. + once = sync.Once{} + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return nil, fmt.Errorf("test error") + } + newC, err := GetOrCreateClient(ctx, "testURI") + assert.Equal(t, oldC, newC) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_Error(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return nil, fmt.Errorf("test error") + } + c, err := GetOrCreateClient(ctx, "testURI") + assert.Nil(t, c) + assert.NotNil(t, err) +} diff --git a/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go b/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go new file mode 100644 index 0000000000..8e6529bfc7 --- /dev/null +++ b/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spinstanceadmin + +import ( + "context" + "fmt" + "sync" + + instance "cloud.google.com/go/spanner/admin/instance/apiv1" +) + +var once sync.Once +var instanceAdminClient *instance.InstanceAdminClient + +// This function is declared as a global variable to make it testable. The unit +// tests edit this function, acting like a double. +var newInstanceAdminClient = instance.NewInstanceAdminClient + +func GetOrCreateClient(ctx context.Context) (*instance.InstanceAdminClient, error) { + var err error + if instanceAdminClient == nil { + once.Do(func() { + instanceAdminClient, err = newInstanceAdminClient(ctx) + }) + if err != nil { + return nil, fmt.Errorf("failed to create spanner instance admin client: %v", err) + } + return instanceAdminClient, nil + } + return instanceAdminClient, nil +} diff --git a/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go b/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go new file mode 100644 index 0000000000..2792d03e82 --- /dev/null +++ b/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spinstanceadmin + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + + instance "cloud.google.com/go/spanner/admin/instance/apiv1" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "google.golang.org/api/option" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func resetTest() { + instanceAdminClient = nil + once = sync.Once{} +} + +func TestGetOrCreateClient_Basic(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newInstanceAdminClient + defer func() { newInstanceAdminClient = oldFunc }() + newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) { + return &instance.InstanceAdminClient{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newInstanceAdminClient + defer func() { newInstanceAdminClient = oldFunc }() + + newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) { + return &instance.InstanceAdminClient{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) + // Explicitly set the client to nil. Running GetOrCreateClient should not create a + // new client since sync would already be executed. + instanceAdminClient = nil + newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) { + return nil, fmt.Errorf("test error") + } + c, err = GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newInstanceAdminClient + defer func() { newInstanceAdminClient = oldFunc }() + + newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) { + return &instance.InstanceAdminClient{}, nil + } + oldC, err := GetOrCreateClient(ctx) + assert.NotNil(t, oldC) + assert.Nil(t, err) + + // Explicitly reset once. Running GetOrCreateClient should not create a + // new client the if condition should prevent it. + once = sync.Once{} + newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) { + return nil, fmt.Errorf("test error") + } + newC, err := GetOrCreateClient(ctx) + assert.Equal(t, oldC, newC) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_Error(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newInstanceAdminClient + defer func() { newInstanceAdminClient = oldFunc }() + + newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) { + return nil, fmt.Errorf("test error") + } + c, err := GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.NotNil(t, err) +} diff --git a/accessors/clients/storage/storage_client.go b/accessors/clients/storage/storage_client.go new file mode 100644 index 0000000000..28c62b868f --- /dev/null +++ b/accessors/clients/storage/storage_client.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package storageclient + +import ( + "context" + "fmt" + "sync" + + "cloud.google.com/go/storage" +) + +var once sync.Once +var gcsClient *storage.Client + +// This function is declared as a global variable to make it testable. The unit +// tests edit this function, acting like a double. +var newClient = storage.NewClient + +func GetOrCreateClient(ctx context.Context) (*storage.Client, error) { + var err error + if gcsClient == nil { + once.Do(func() { + gcsClient, err = newClient(ctx) + }) + if err != nil { + return nil, fmt.Errorf("failed to create storage client: %v", err) + } + return gcsClient, nil + } + return gcsClient, nil +} diff --git a/accessors/clients/storage/storage_client_test.go b/accessors/clients/storage/storage_client_test.go new file mode 100644 index 0000000000..73bd6b873a --- /dev/null +++ b/accessors/clients/storage/storage_client_test.go @@ -0,0 +1,117 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package storageclient + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + + "cloud.google.com/go/storage" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "google.golang.org/api/option" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func resetTest() { + gcsClient = nil + once = sync.Once{} +} + +func TestGetOrCreateClient_Basic(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) { + return &storage.Client{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) { + return &storage.Client{}, nil + } + c, err := GetOrCreateClient(ctx) + assert.NotNil(t, c) + assert.Nil(t, err) + // Explicitly set the client to nil. Running GetOrCreateClient should not create a + // new client since sync would already be executed. + gcsClient = nil + + newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) { + return nil, fmt.Errorf("test error") + } + c, err = GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) { + return &storage.Client{}, nil + } + oldC, err := GetOrCreateClient(ctx) + assert.NotNil(t, oldC) + assert.Nil(t, err) + + // Explicitly reset once. Running GetOrCreateClient should not create a + // new client the if condition should prevent it. + once = sync.Once{} + newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) { + return nil, fmt.Errorf("test error") + } + newC, err := GetOrCreateClient(ctx) + assert.Equal(t, oldC, newC) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_Error(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) { + return nil, fmt.Errorf("test error") + } + c, err := GetOrCreateClient(ctx) + assert.Nil(t, c) + assert.NotNil(t, err) +} diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go new file mode 100644 index 0000000000..3cffedaaba --- /dev/null +++ b/accessors/dataflow/dataflow_accessor.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dataflowaccessor + +import ( + "context" + "fmt" + + "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb" + dataflowclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/dataflow" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/googleapis/gax-go/v2" +) + +type DataflowAccessor interface { + LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) +} + +type DataflowAccessorImpl struct{} + +func (dfA *DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) { + dfClient, err := dataflowclient.GetOrCreateClient(ctx) + if err != nil { + return nil, err + } + respDf, err := dfClient.LaunchFlexTemplate(ctx, req) + if err != nil { + logger.Log.Error(fmt.Sprintf("flexTemplateRequest: %+v\n", req)) + return nil, fmt.Errorf("error launching dataflow template: %v", err) + } + return respDf, nil +} diff --git a/accessors/dataflow/dataflow_accessor_test.go b/accessors/dataflow/dataflow_accessor_test.go new file mode 100644 index 0000000000..9b9ed5a4e7 --- /dev/null +++ b/accessors/dataflow/dataflow_accessor_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dataflowaccessor diff --git a/accessors/dataflow/dataflow_types.go b/accessors/dataflow/dataflow_types.go new file mode 100644 index 0000000000..d7b8f355ad --- /dev/null +++ b/accessors/dataflow/dataflow_types.go @@ -0,0 +1,32 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dataflowaccessor + +type DataflowTuningConfig struct { + ProjectId string `json:"projectId"` + JobName string `json:"jobName"` + Location string `json:"location"` + VpcHostProjectId string `json:"hostProjectId"` + Network string `json:"network"` + Subnetwork string `json:"subnetwork"` + MaxWorkers int32 `json:"maxWorkers"` + NumWorkers int32 `json:"numWorkers"` + ServiceAccountEmail string `json:"serviceAccountEmail"` + MachineType string `json:"machineType"` + AdditionalUserLabels map[string]string `json:"additionalUserLabels"` + KmsKeyName string `json:"kmsKeyName"` + GcsTemplatePath string `json:"gcsTemplatePath"` + AdditionalExperiments []string `json:"additionalExperiments"` + EnableStreamingEngine bool `json:"enableStreamingEngine"` +} diff --git a/accessors/spanner/spanner_accessor.go b/accessors/spanner/spanner_accessor.go new file mode 100644 index 0000000000..a5972ae036 --- /dev/null +++ b/accessors/spanner/spanner_accessor.go @@ -0,0 +1,211 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spanneraccessor + +import ( + "context" + "fmt" + "strings" + "time" + + "cloud.google.com/go/spanner" + "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" + "cloud.google.com/go/spanner/admin/instance/apiv1/instancepb" + spanneradmin "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/admin" + spannerclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/client" + spinstanceadmin "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/instanceadmin" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" + "google.golang.org/api/iterator" +) + +type SpannerAccessor interface { + GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) + GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) + CheckExistingDb(ctx context.Context, dbURI string) (bool, error) + CreateEmptyDatabase(ctx context.Context, dbURI string) error + GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) + CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) + ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error + CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error +} + +type SpannerAccessorImpl struct{} + +func (sp *SpannerAccessorImpl) GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) { + adminClient, err := spanneradmin.GetOrCreateClient(ctx) + if err != nil { + return nil, err + } + return adminClient.GetDatabase(ctx, &databasepb.GetDatabaseRequest{Name: dbURI}) +} + +func (sp *SpannerAccessorImpl) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) { + result, err := sp.GetDatabase(ctx, dbURI) + if err != nil { + return "", fmt.Errorf("cannot connect to database: %v", err) + } + return strings.ToLower(result.DatabaseDialect.String()), nil +} + +// CheckExistingDb checks whether the database with dbURI exists or not. +// If API call doesn't respond then user is informed after every 5 minutes on command line. +func (sp *SpannerAccessorImpl) CheckExistingDb(ctx context.Context, dbURI string) (bool, error) { + gotResponse := make(chan bool) + var err error + go func() { + _, err = sp.GetDatabase(ctx, dbURI) + gotResponse <- true + }() + for { + select { + case <-time.After(5 * time.Minute): + fmt.Println("WARNING! API call not responding: make sure that spanner api endpoint is configured properly") + case <-gotResponse: + if err != nil { + if utils.ContainsAny(strings.ToLower(err.Error()), []string{"database not found"}) { + return false, nil + } + return false, fmt.Errorf("can't get database info: %s", err) + } + return true, nil + } + } +} + +func (sp *SpannerAccessorImpl) CreateEmptyDatabase(ctx context.Context, dbURI string) error { + adminClient, err := spanneradmin.GetOrCreateClient(ctx) + if err != nil { + return err + } + project, instance, dbName := utils.ParseDbURI(dbURI) + req := &databasepb.CreateDatabaseRequest{ + Parent: fmt.Sprintf("projects/%s/instances/%s", project, instance), + CreateStatement: "CREATE DATABASE `" + dbName + "`", + } + op, err := adminClient.CreateDatabase(ctx, req) + if err != nil { + return fmt.Errorf("can't build CreateDatabaseRequest: %w", utils.AnalyzeError(err, dbURI)) + } + if _, err := op.Wait(ctx); err != nil { + return fmt.Errorf("createDatabase call failed: %w", utils.AnalyzeError(err, dbURI)) + } + return nil +} + +func (sp *SpannerAccessorImpl) GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) { + instanceClient, err := spinstanceadmin.GetOrCreateClient(ctx) + if err != nil { + return "", err + } + instanceInfo, err := instanceClient.GetInstance(ctx, &instancepb.GetInstanceRequest{Name: instanceURI}) + if err != nil { + return "", err + } + instanceConfig, err := instanceClient.GetInstanceConfig(ctx, &instancepb.GetInstanceConfigRequest{Name: instanceInfo.Config}) + if err != nil { + return "", err + + } + for _, replica := range instanceConfig.Replicas { + if replica.DefaultLeaderLocation { + return replica.Location, nil + } + } + return "", fmt.Errorf("no leader found for spanner instance %s while trying fetch location", instanceURI) +} + +func (sp *SpannerAccessorImpl) CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) { + spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI) + if err != nil { + return false, err + } + stmt := spanner.Statement{ + SQL: `SELECT CHANGE_STREAM_NAME FROM information_schema.change_streams`, + } + iter := spClient.Single().Query(ctx, stmt) + defer iter.Stop() + var cs_name string + csExists := false + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return false, fmt.Errorf("couldn't read row from change_streams table: %w", err) + } + err = row.Columns(&cs_name) + if err != nil { + return false, fmt.Errorf("can't scan row from change_streams table: %v", err) + } + if cs_name == changeStreamName { + csExists = true + break + } + } + return csExists, nil +} + +func (sp *SpannerAccessorImpl) ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error { + spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI) + if err != nil { + return err + } + // Validate if change stream options are set correctly. + stmt := spanner.Statement{ + SQL: `SELECT option_value FROM information_schema.change_stream_options + WHERE change_stream_name = @p1 AND option_name = 'value_capture_type'`, + Params: map[string]interface{}{ + "p1": changeStreamName, + }, + } + iter := spClient.Single().Query(ctx, stmt) + defer iter.Stop() + var option_value string + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("couldn't read row from change_stream_options table: %w", err) + } + err = row.Columns(&option_value) + if err != nil { + return fmt.Errorf("can't scan row from change_stream_options table: %v", err) + } + if option_value != "NEW_ROW" { + return fmt.Errorf("VALUE_CAPTURE_TYPE for changestream %s is not NEW_ROW. Please update the changestream option or create a new one", changeStreamName) + } + } + return nil +} + +func (sp *SpannerAccessorImpl) CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error { + spClient, _ := spanneradmin.GetOrCreateClient(ctx) + op, err := spClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ + Database: dbURI, + // TODO: create change stream for only the tables present in Spanner. + Statements: []string{fmt.Sprintf("CREATE CHANGE STREAM %s FOR ALL OPTIONS (value_capture_type = 'NEW_ROW', retention_period = '7d')", changeStreamName)}, + }) + if err != nil { + return fmt.Errorf("cannot submit request create change stream request: %v", err) + } + if err := op.Wait(ctx); err != nil { + return fmt.Errorf("could not update database ddl: %v", err) + } else { + fmt.Println("Successfully created changestream", changeStreamName) + } + return nil +} diff --git a/accessors/spanner/spanner_accessor_test.go b/accessors/spanner/spanner_accessor_test.go new file mode 100644 index 0000000000..5f0d2f60b9 --- /dev/null +++ b/accessors/spanner/spanner_accessor_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package spanneraccessor diff --git a/accessors/storage/storage_accessor.go b/accessors/storage/storage_accessor.go new file mode 100644 index 0000000000..dc0d404bb3 --- /dev/null +++ b/accessors/storage/storage_accessor.go @@ -0,0 +1,211 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package storageaccessor + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "cloud.google.com/go/storage" + storageclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/storage" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "google.golang.org/api/googleapi" +) + +type StorageAccessor interface { + CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error + CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error + EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error + UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error + WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error + ReadGcsFile(ctx context.Context, filePath string) (string, error) + ReadAnyFile(ctx context.Context, filePath string) (string, error) +} + +type StorageAccessorImpl struct{} + +func (sa *StorageAccessorImpl) CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error { + return sa.createGCSBucketUtil(ctx, bucketName, projectID, location, nil, 0) +} + +func (sa *StorageAccessorImpl) CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error { + return sa.createGCSBucketUtil(ctx, bucketName, projectID, location, matchesPrefix, ttl) +} + +func (sa *StorageAccessorImpl) createGCSBucketUtil(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error { + client, err := storageclient.GetOrCreateClient(ctx) + if err != nil { + return err + } + bucket := client.Bucket(bucketName) + attrs := storage.BucketAttrs{ + Location: location, + } + if ttl > 0 { + attrs.Lifecycle = storage.Lifecycle{ + Rules: []storage.LifecycleRule{ + { + Action: storage.LifecycleAction{Type: "Delete"}, + Condition: storage.LifecycleCondition{ + AgeInDays: ttl, + // The prefixes should not contain the bucket names and starting slash. + // For object gs://my_bucket/pictures/paris_2022.jpg, + // you would use a condition such as "matchesPrefix":["pictures/paris_"]. + MatchesPrefix: matchesPrefix, + }, + }, + }, + } + } + + if err := bucket.Create(ctx, projectID, &attrs); err != nil { + if e, ok := err.(*googleapi.Error); ok { + // Ignoring the bucket already exists error. + if e.Code != 409 { + return fmt.Errorf("failed to create bucket: %v", err) + } else { + fmt.Printf("Using the existing bucket: %v \n", bucketName) + } + } else { + return fmt.Errorf("failed to create bucket: %v", err) + } + + } else { + logger.Log.Info(fmt.Sprintf("Created new GCS bucket: %v\n", bucketName)) + } + return nil +} + +// Applies the bucket lifecycle with delete rule. Only accepts the Age and +// prefix rule conditions as it is only used for the Datastream destination +// bucket currently. +func (sa *StorageAccessorImpl) EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error { + client, err := storageclient.GetOrCreateClient(ctx) + if err != nil { + return fmt.Errorf("could not create client while enabling lifecycle: %w", err) + } + + for i, str := range matchesPrefix { + matchesPrefix[i] = strings.TrimPrefix(str, "/") + } + bucket := client.Bucket(bucketName) + bucketAttrsToUpdate := storage.BucketAttrsToUpdate{ + Lifecycle: &storage.Lifecycle{ + Rules: []storage.LifecycleRule{ + { + Action: storage.LifecycleAction{Type: "Delete"}, + Condition: storage.LifecycleCondition{ + AgeInDays: ttl, + // The prefixes should not contain the bucket names and starting slash. + // For object gs://my_bucket/pictures/paris_2022.jpg, + // you would use a condition such as "matchesPrefix":["pictures/paris_"]. + MatchesPrefix: matchesPrefix, + }, + }, + }, + }, + } + + attrs, err := bucket.Update(ctx, bucketAttrsToUpdate) + if err != nil { + return fmt.Errorf("could not bucket with lifecycle: %w", err) + } + logger.Log.Info(fmt.Sprintf("Added lifecycle rule to bucket %v\n. Rule Action: %v\t Rule Condition: %v\n", + bucketName, attrs.Lifecycle.Rules[0].Action, attrs.Lifecycle.Rules[0].Condition)) + return nil +} + +// UploadLocalFileToGCS uploads an object. +func (sa *StorageAccessorImpl) UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error { + data, err := os.ReadFile(localFilePath) + if err != nil { + return fmt.Errorf("could not read file %s: %w", localFilePath, err) + } + return sa.WriteDataToGCS(ctx, filePath, fileName, string(data)) +} + +func (sa *StorageAccessorImpl) WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error { + client, err := storageclient.GetOrCreateClient(ctx) + if err != nil { + return fmt.Errorf("could not create client while uploading to GCS: %w", err) + } + + u, err := utils.ParseGCSFilePath(filePath) + if err != nil { + return fmt.Errorf("parseFilePath: unable to parse file path: %v", err) + } + bucketName := u.Host + bucket := client.Bucket(bucketName) + obj := bucket.Object(u.Path[1:] + fileName) + + w := obj.NewWriter(ctx) + logger.Log.Info(fmt.Sprintf("Writing data to %s", filePath)) + n, err := fmt.Fprint(w, data) + if err != nil { + fmt.Printf("Failed to write to Cloud Storage: %s", filePath) + return err + } + logger.Log.Info(fmt.Sprintf("Wrote %d bytes to GCS", n)) + + if err := w.Close(); err != nil { + fmt.Printf("Failed to close GCS file: %s", filePath) + return err + } + return nil +} + +func (sa *StorageAccessorImpl) ReadGcsFile(ctx context.Context, filePath string) (string, error) { + client, err := storageclient.GetOrCreateClient(ctx) + if err != nil { + return "", fmt.Errorf("could not create client: %w", err) + } + + u, err := utils.ParseGCSFilePath(filePath) + if err != nil { + return "", fmt.Errorf("unable to parse file path: %v", err) + } + bucketName := u.Host + bucket := client.Bucket(bucketName) + obj := bucket.Object(u.Path[1:]) + + rc, err := obj.NewReader(ctx) + if err != nil { + return "", err + } + defer rc.Close() + buf := new(strings.Builder) + logger.Log.Info(fmt.Sprintf("Reading from %s", filePath)) + n, err := io.Copy(buf, rc) + if err != nil { + return "", err + } + logger.Log.Info(fmt.Sprintf("Read %d bytes", n)) + return buf.String(), nil +} + +func (sa *StorageAccessorImpl) ReadAnyFile(ctx context.Context, filePath string) (string, error) { + if strings.HasPrefix(filePath, constants.GCS_FILE_PREFIX) { + return sa.ReadGcsFile(ctx, filePath) + } + buf, err := os.ReadFile(filePath) + if err != nil { + return "", err + } + return string(buf), nil +} diff --git a/accessors/storage/storage_accessor_test.go b/accessors/storage/storage_accessor_test.go new file mode 100644 index 0000000000..1519db317b --- /dev/null +++ b/accessors/storage/storage_accessor_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package storageaccessor diff --git a/common/utils/dataflow_utils.go b/accessors/utils/dataflow/dataflow_utils.go similarity index 52% rename from common/utils/dataflow_utils.go rename to accessors/utils/dataflow/dataflow_utils.go index a5d4ac09f9..2a6a9f69b9 100644 --- a/common/utils/dataflow_utils.go +++ b/accessors/utils/dataflow/dataflow_utils.go @@ -1,10 +1,10 @@ -// Copyright 2023 Google LLC +// Copyright 2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,20 +12,66 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package utils contains common helper functions used across multiple other packages. -// Utils should not import any Spanner migration tool packages. -package utils +// This is a package is kept with accessors because some functions import other accessors. +// The common/utils package should not import any SMT dependency. +package dataflowutils import ( + "context" "encoding/json" "fmt" "sort" "strings" "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb" + dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" "golang.org/x/exp/maps" ) +func GetDataflowLaunchRequest(parameters map[string]string, cfg dataflowaccessor.DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) { + // If custom network is not selected, use public IP. Typical for internal testing flow. + vpcSubnetwork := "" + workerIpAddressConfig := dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PUBLIC + if cfg.Network != "" || cfg.Subnetwork != "" { + workerIpAddressConfig = dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE + // If subnetwork is not provided, assume network has auto subnet configuration. + if cfg.Subnetwork != "" { + if cfg.VpcHostProjectId == "" || cfg.Location == "" { + return nil, fmt.Errorf("vpc host project id and location must be specified when specifying subnetwork") + } + vpcSubnetwork = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", cfg.VpcHostProjectId, cfg.Location, cfg.Subnetwork) + } + } + // Dataflow does not accept upper case letters in the name. + cfg.JobName = strings.ToLower(cfg.JobName) + request := &dataflowpb.LaunchFlexTemplateRequest{ + ProjectId: cfg.ProjectId, + LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{ + JobName: cfg.JobName, + Template: &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: cfg.GcsTemplatePath}, + Parameters: parameters, + Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{ + MaxWorkers: cfg.MaxWorkers, + NumWorkers: cfg.NumWorkers, + ServiceAccountEmail: cfg.ServiceAccountEmail, + MachineType: cfg.MachineType, + AdditionalUserLabels: cfg.AdditionalUserLabels, + KmsKeyName: cfg.KmsKeyName, + Network: cfg.Network, + Subnetwork: vpcSubnetwork, + IpConfiguration: workerIpAddressConfig, + AdditionalExperiments: cfg.AdditionalExperiments, + EnableStreamingEngine: cfg.EnableStreamingEngine, + }, + }, + Location: cfg.Location, + } + logger.Log.Debug(fmt.Sprintf("Flex Template request generated: %+v", request)) + return request, nil +} + // Generate the equivalent gCloud CLI command to launch a dataflow job with the same parameters and environment flags // as the input body. func GetGcloudDataflowCommand(req *dataflowpb.LaunchFlexTemplateRequest) string { @@ -80,11 +126,7 @@ func getEnvironmentFlags(environment *dataflowpb.FlexTemplateRuntimeEnvironment) flag += fmt.Sprintf("--subnetwork %s ", environment.Subnetwork) } if environment.AdditionalUserLabels != nil && len(environment.AdditionalUserLabels) > 0 { - jsonByteStr, err := json.Marshal(environment.AdditionalUserLabels) - // If error is not nil, omit this flag and move on. We don't need error handling here. - if err == nil { - flag += fmt.Sprintf("--additional-user-labels %s ", string(jsonByteStr)) - } + flag += fmt.Sprintf("--additional-user-labels %s ", formatAdditionalUserLabels(environment.AdditionalUserLabels)) } if environment.KmsKeyName != "" { flag += fmt.Sprintf("--dataflow-kms-key %s ", environment.KmsKeyName) @@ -102,10 +144,31 @@ func getEnvironmentFlags(environment *dataflowpb.FlexTemplateRuntimeEnvironment) flag += "--enable-streaming-engine " } if environment.FlexrsGoal != dataflowpb.FlexResourceSchedulingGoal_FLEXRS_UNSPECIFIED { - flag += fmt.Sprintf("--flexrs-goal %s ", environment.FlexrsGoal) + flag += fmt.Sprintf("--flexrs-goal %s ", environment.FlexrsGoal) } if environment.StagingLocation != "" { flag += fmt.Sprintf("--staging-location %s ", environment.StagingLocation) } return strings.Trim(flag, " ") } + +func formatAdditionalUserLabels(labels map[string]string) string { + res := []string{} + for key, value := range labels { + res = append(res, fmt.Sprintf("%s=%s", key, value)) + } + return strings.Join(res, ",") +} + +func UnmarshalDataflowTuningConfig(ctx context.Context, sa storageaccessor.StorageAccessor, filePath string) (dataflowaccessor.DataflowTuningConfig, error) { + jsonStr, err := sa.ReadAnyFile(ctx, filePath) + if err != nil { + return dataflowaccessor.DataflowTuningConfig{}, err + } + tuningCfg := dataflowaccessor.DataflowTuningConfig{} + err = json.Unmarshal([]byte(jsonStr), &tuningCfg) + if err != nil { + return dataflowaccessor.DataflowTuningConfig{}, err + } + return tuningCfg, nil +} diff --git a/accessors/utils/dataflow/dataflow_utils_test.go b/accessors/utils/dataflow/dataflow_utils_test.go new file mode 100644 index 0000000000..2d1e33ceff --- /dev/null +++ b/accessors/utils/dataflow/dataflow_utils_test.go @@ -0,0 +1,333 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dataflowutils + +import ( + "context" + "fmt" + "os" + "testing" + + "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb" + dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func getParameters() map[string]string { + return map[string]string{ + "inputFilePattern": "gs://inputFilePattern", + "streamName": "my-stream", + "instanceId": "my-instance", + "databaseId": "my-dbName", + "sessionFilePath": "gs://session.json", + "deadLetterQueueDirectory": "gs://dlq", + "transformationContextFilePath": "gs://transformationContext.json", + "directoryWatchDurationInMinutes": "480", // Setting directory watch timeout to 8 hours + } +} + +func getTuningConfig() dataflowaccessor.DataflowTuningConfig { + return dataflowaccessor.DataflowTuningConfig{ + ProjectId: "test-project", + JobName: "test-job", + Location: "us-central1", + VpcHostProjectId: "host-project", + Network: "my-network", + Subnetwork: "my-subnetwork", + MaxWorkers: 50, + NumWorkers: 10, + ServiceAccountEmail: "svc-account@google.com", + MachineType: "n2-standard-64", + AdditionalUserLabels: map[string]string{"name": "wrench"}, + KmsKeyName: "sample-kms-key", + GcsTemplatePath: "gs://template/Cloud_Datastream_to_Spanner", + AdditionalExperiments: []string{"use_runner_V2", "test-experiment"}, + EnableStreamingEngine: true, + } +} + +func getTemplateDfRequest1() *dataflowpb.LaunchFlexTemplateRequest { + return &dataflowpb.LaunchFlexTemplateRequest{ + ProjectId: "test-project", + Location: "us-central1", + LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{ + JobName: "test-job", + Template: &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://template/Cloud_Datastream_to_Spanner"}, + Parameters: getParameters(), + Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{ + MaxWorkers: 50, + NumWorkers: 10, + ServiceAccountEmail: "svc-account@google.com", + MachineType: "n2-standard-64", + AdditionalUserLabels: map[string]string{"name": "wrench"}, + KmsKeyName: "sample-kms-key", + Network: "my-network", + Subnetwork: "https://www.googleapis.com/compute/v1/projects/host-project/regions/us-central1/subnetworks/my-subnetwork", + IpConfiguration: dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE, + AdditionalExperiments: []string{"use_runner_V2", "test-experiment"}, + EnableStreamingEngine: true, + }, + }, + } +} + +func TestGetDataflowLaunchRequestBasic(t *testing.T) { + params := getParameters() + cfg := getTuningConfig() + actual, err := GetDataflowLaunchRequest(params, cfg) + if err != nil { + t.Fail() + } + expected := getTemplateDfRequest1() + assert.True(t, EquateLaunchFlexTemplateRequest(expected, actual)) +} + +func TestGetDataflowLaunchRequestMissingVpcHost(t *testing.T) { + params := getParameters() + cfg := getTuningConfig() + cfg.VpcHostProjectId = "" + _, err := GetDataflowLaunchRequest(params, cfg) + assert.True(t, err != nil) +} + +func TestGetDataflowLaunchRequestNameToLowerCase(t *testing.T) { + params := getParameters() + cfg := getTuningConfig() + cfg.JobName = "CAPITalJobName" + actual, err := GetDataflowLaunchRequest(params, cfg) + if err != nil { + t.Fail() + } + expected := getTemplateDfRequest1() + expected.LaunchParameter.JobName = "capitaljobname" + assert.True(t, EquateLaunchFlexTemplateRequest(expected, actual)) +} + +func getTemplateDfRequest2() *dataflowpb.LaunchFlexTemplateRequest { + launchParameters := &dataflowpb.LaunchFlexTemplateParameter{ + JobName: "test-job", + Template: &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://template/Cloud_Datastream_to_Spanner"}, + Parameters: getParameters(), + Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{ + MaxWorkers: 50, + NumWorkers: 10, + ServiceAccountEmail: "svc-account@google.com", + TempLocation: "gs://temp-location", + MachineType: "n2-standard-64", + AdditionalExperiments: []string{"use_runner_V2", "test-experiment"}, + Network: "my-network", + Subnetwork: "https://www.googleapis.com/compute/v1/projects/host-project/regions/us-central1/subnetworks/my-subnetwork", + AdditionalUserLabels: map[string]string{"name": "wrench"}, + KmsKeyName: "sample-kms-key", + IpConfiguration: dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE, + WorkerRegion: "test-worker-region", + WorkerZone: "test-worker-zone", + EnableStreamingEngine: true, + FlexrsGoal: 1, + StagingLocation: "gs://staging-location", + }, + } + req := &dataflowpb.LaunchFlexTemplateRequest{ + ProjectId: "test-project", + LaunchParameter: launchParameters, + Location: "us-central1", + } + return req +} + +func TestGcloudCmdWithAllParams(t *testing.T) { + + req := getTemplateDfRequest2() + expectedCmd := "gcloud dataflow flex-template run test-job " + + "--project=test-project --region=us-central1 " + + "--template-file-gcs-location=gs://template/Cloud_Datastream_to_Spanner " + + "--num-workers 10 --max-workers 50 --service-account-email svc-account@google.com " + + "--temp-location gs://temp-location --worker-machine-type n2-standard-64 " + + "--additional-experiments use_runner_V2,test-experiment --network my-network " + + "--subnetwork https://www.googleapis.com/compute/v1/projects/host-project/regions/us-central1/subnetworks/my-subnetwork --additional-user-labels name=wrench " + + "--dataflow-kms-key sample-kms-key --disable-public-ips --worker-region test-worker-region " + + "--worker-zone test-worker-zone --enable-streaming-engine " + + "--flexrs-goal FLEXRS_SPEED_OPTIMIZED --staging-location gs://staging-location " + + "--parameters databaseId=my-dbName,deadLetterQueueDirectory=gs://dlq," + + "directoryWatchDurationInMinutes=480,inputFilePattern=gs://inputFilePattern," + + "instanceId=my-instance,sessionFilePath=gs://session.json,streamName=my-stream," + + "transformationContextFilePath=gs://transformationContext.json" + assert.Equal(t, expectedCmd, GetGcloudDataflowCommand(req)) +} + +func TestGcloudCmdWithPartialParams(t *testing.T) { + + req := getTemplateDfRequest2() + req.LaunchParameter.Parameters = make(map[string]string) + req.LaunchParameter.Environment.FlexrsGoal = 0 + req.LaunchParameter.Environment.IpConfiguration = 0 + req.LaunchParameter.Environment.EnableStreamingEngine = false + req.LaunchParameter.Environment.AdditionalExperiments = []string{} + req.LaunchParameter.Environment.AdditionalUserLabels = make(map[string]string) + req.LaunchParameter.Environment.WorkerRegion = "" + req.LaunchParameter.Environment.NumWorkers = 0 + req.LaunchParameter.Environment.Network = "" + req.LaunchParameter.Environment.Subnetwork = "" + + expectedCmd := "gcloud dataflow flex-template run test-job " + + "--project=test-project --region=us-central1 " + + "--template-file-gcs-location=gs://template/Cloud_Datastream_to_Spanner " + + "--max-workers 50 --service-account-email svc-account@google.com " + + "--temp-location gs://temp-location --worker-machine-type n2-standard-64 " + + "--dataflow-kms-key sample-kms-key " + + "--worker-zone test-worker-zone " + + "--staging-location gs://staging-location" + assert.Equal(t, expectedCmd, GetGcloudDataflowCommand(req)) +} + +func EquateLaunchFlexTemplateRequest(df1 *dataflowpb.LaunchFlexTemplateRequest, df2 *dataflowpb.LaunchFlexTemplateRequest) bool { + lp1 := df1.LaunchParameter + lp2 := df2.LaunchParameter + return (df1.ProjectId == df2.ProjectId && + df1.Location == df2.Location && + lp1.JobName == lp2.JobName && + lp1.Environment.MaxWorkers == lp2.Environment.MaxWorkers && + lp1.Environment.NumWorkers == lp2.Environment.NumWorkers && + lp1.Environment.ServiceAccountEmail == lp2.Environment.ServiceAccountEmail && + lp1.Environment.MachineType == lp2.Environment.MachineType && + lp1.Environment.KmsKeyName == lp2.Environment.KmsKeyName && + lp1.Environment.Network == lp2.Environment.Network && + lp1.Environment.Subnetwork == lp2.Environment.Subnetwork && + lp1.Environment.GetIpConfiguration().String() == lp2.Environment.GetIpConfiguration().String() && + lp1.Environment.EnableStreamingEngine == lp2.Environment.EnableStreamingEngine && + cmp.Equal(lp1.Environment.AdditionalUserLabels, lp2.Environment.AdditionalUserLabels) && + cmp.Equal(lp1.Environment.AdditionalExperiments, lp2.Environment.AdditionalExperiments) && + lp1.GetContainerSpecGcsPath() == lp2.GetContainerSpecGcsPath()) +} + +type StorageAccessorMock struct { + storageaccessor.StorageAccessorImpl +} + +var readAnyFileMock func(ctx context.Context, filePath string) (string, error) + +func (sam StorageAccessorMock) ReadAnyFile(ctx context.Context, filePath string) (string, error) { + return readAnyFileMock(ctx, filePath) +} + +func TestUnmarshalDataflowTuningConfig(t *testing.T) { + testCases := []struct { + name string + readAnyFileMock func(ctx context.Context, filePath string) (string, error) + expectError bool + want dataflowaccessor.DataflowTuningConfig + }{ + { + name: "Basic", + readAnyFileMock: func(ctx context.Context, filePath string) (string, error) { + return `{ + "projectId": "test-project", + "jobName": "test-job-name", + "location": "us-central1", + "network": "test-network", + "subnetwork": "test-subnetwork", + "hostProjectId": "test-host-project", + "maxWorkers": 3, + "numWorkers": 2, + "serviceAccountEmail": "abc@xyz.com", + "machineType": "n1-standard-8", + "additionalUserLabels": {"my": "label"}, + "kmsKeyName": "test-key", + "gcsTemplatePath": "gs://path", + "additionalExperiments": ["xyz","123"], + "enableStreamingEngine": true + }`, nil + }, + expectError: false, + want: dataflowaccessor.DataflowTuningConfig{ + ProjectId: "test-project", + JobName: "test-job-name", + Location: "us-central1", + Network: "test-network", + Subnetwork: "test-subnetwork", + VpcHostProjectId: "test-host-project", + MaxWorkers: 3, + NumWorkers: 2, + ServiceAccountEmail: "abc@xyz.com", + MachineType: "n1-standard-8", + AdditionalUserLabels: map[string]string{"my": "label"}, + KmsKeyName: "test-key", + GcsTemplatePath: "gs://path", + AdditionalExperiments: []string{"xyz", "123"}, + EnableStreamingEngine: true, + }, + }, + { + name: "Defaults", + readAnyFileMock: func(ctx context.Context, filePath string) (string, error) { + return `{}`, nil + }, + expectError: false, + want: dataflowaccessor.DataflowTuningConfig{ + ProjectId: "", + JobName: "", + Location: "", + Network: "", + Subnetwork: "", + VpcHostProjectId: "", + MaxWorkers: 0, + NumWorkers: 0, + ServiceAccountEmail: "", + MachineType: "", + AdditionalUserLabels: nil, + KmsKeyName: "", + GcsTemplatePath: "", + AdditionalExperiments: nil, + EnableStreamingEngine: false, + }, + }, + { + name: "ReadAnyFile throws error", + readAnyFileMock: func(ctx context.Context, filePath string) (string, error) { + return "", fmt.Errorf("test error") + }, + expectError: true, + want: dataflowaccessor.DataflowTuningConfig{}, + }, + { + name: "Json unmarshall throws error", + readAnyFileMock: func(ctx context.Context, filePath string) (string, error) { + return "{\"abc\"", nil + }, + expectError: true, + want: dataflowaccessor.DataflowTuningConfig{}, + }, + } + ctx := context.Background() + saMock := StorageAccessorMock{} + for _, tc := range testCases { + readAnyFileMock = tc.readAnyFileMock + got, err := UnmarshalDataflowTuningConfig(ctx, &saMock, "unused/path/due/to/mock") + assert.Equal(t, tc.expectError, err != nil) + assert.Equal(t, tc.want, got) + } +} diff --git a/activity/IActivity.go b/activity/IActivity.go new file mode 100644 index 0000000000..245ec2e8e1 --- /dev/null +++ b/activity/IActivity.go @@ -0,0 +1,21 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import "context" + +type Activity interface { + Transaction(ctx context.Context) error + Compensation(ctx context.Context) error +} diff --git a/cmd/data.go b/cmd/data.go index c908f8ad94..b31f041269 100644 --- a/cmd/data.go +++ b/cmd/data.go @@ -26,6 +26,7 @@ import ( sp "cloud.google.com/go/spanner" database "cloud.google.com/go/spanner/admin/database/apiv1" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" "github.com/GoogleCloudPlatform/spanner-migration-tool/conversion" @@ -178,7 +179,8 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface // validateExistingDb validates that the existing spanner schema is in accordance with the one specified in the session file. func validateExistingDb(ctx context.Context, spDialect, dbURI string, adminClient *database.DatabaseAdminClient, client *sp.Client, conv *internal.Conv) error { - dbExists, err := conversion.CheckExistingDb(ctx, adminClient, dbURI) + spA := spanneraccessor.SpannerAccessorImpl{} + dbExists, err := spA.CheckExistingDb(ctx, dbURI) if err != nil { err = fmt.Errorf("can't verify target database: %v", err) return err diff --git a/common/constants/constants.go b/common/constants/constants.go index e98855a81d..11118fc477 100644 --- a/common/constants/constants.go +++ b/common/constants/constants.go @@ -64,7 +64,8 @@ const ( MigrationMetadataKey string = "cloud-spanner-migration-metadata" // Scheme used for GCS paths - GCS_SCHEME string = "gs" + GCS_SCHEME string = "gs" + GCS_FILE_PREFIX string = "gs://" // File upload prefix for dump and session load. UPLOAD_FILE_DIR string = "upload-file" @@ -100,4 +101,22 @@ const ( // Metadata table names SMT_JOB_TABLE string = "SMT_JOB" SMT_RESOURCE_TABLE string = "SMT_RESOURCE" + + // Reverse Replication + REVERSE_REPLICATION_JOB_TYPE string = "reverse-replication" + REVERSE_REPLICATION_READER_TEMPLATE_PATH string = "gs://dataflow-templates-us-central2/2024-01-09-00_RC01/flex/Spanner_Change_Streams_to_Sharded_File_Sink" + REVERSE_REPLICATION_WRITER_TEMPLATE_PATH string = "gs://dataflow-templates-us-central2/2024-01-09-00_RC01/flex/GCS_to_Sourcedb" + + // Reverse replication - Reader template. + RR_READER_FILTER_NONE string = "none" + RR_READER_FILTER_FWD string = "forward_migration" + RR_READER_REGULAR_MODE string = "regular" + RR_READER_RESUME_MODE string = "resume" + + // Reverse replication - Writer template. + RR_WRITER_REGULAR_MODE string = "regular" + RR_WRITER_REPROCESS_MODE string = "reprocess" + RR_WRITER_RESUME_SUCCESS_MODE string = "resumeSuccess" + RR_WRITER_RESUME_FAILED_MODE string = "resumeFailed" + RR_WRITER_RESUME_ALL_MODE string = "resumeAll" ) diff --git a/common/metrics/dashboard_components.go b/common/metrics/dashboard_components.go index dcd55b3358..b0e01a530b 100644 --- a/common/metrics/dashboard_components.go +++ b/common/metrics/dashboard_components.go @@ -4,16 +4,13 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -// Package utils contains common helper functions used across multiple other packages. -// Utils should not import any Spanner migration tool packages. package metrics import ( @@ -42,22 +39,22 @@ var dashboardClient *dashboard.DashboardsClient // MonitoringMetricsResources contains information required to create the monitoring dashboard type MonitoringMetricsResources struct { - ProjectId string - DataflowJobId string - DatastreamId string - JobMetadataGcsBucket string - PubsubSubscriptionId string - SpannerInstanceId string - SpannerDatabaseId string - ShardToShardResourcesMap map[string]internal.ShardResources - ShardId string + ProjectId string + DataflowJobId string + DatastreamId string + JobMetadataGcsBucket string + PubsubSubscriptionId string + SpannerInstanceId string + SpannerDatabaseId string + ShardToShardResourcesMap map[string]internal.ShardResources + ShardId string MigrationRequestId string } type TileInfo struct { Title string TimeSeriesQueries map[string]string // Map of legend template and their corresponding queries - TextContent string // string for text input + TextContent string // string for text input } type MosaicGroup struct { @@ -95,10 +92,10 @@ func createShardDataflowMetrics(resourceIds MonitoringMetricsResources) []*dashb TileInfo{ Title: "Dataflow Workers Memory Utilization", TimeSeriesQueries: map[string]string{ - "p50 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "50"), - "p90 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "90"), - "Max worker": fmt.Sprintf(dataflowMemoryUtilMaxQuery, resourceIds.DataflowJobId), - }}.createXYChartTile(), + "p50 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "50"), + "p90 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "90"), + "Max worker": fmt.Sprintf(dataflowMemoryUtilMaxQuery, resourceIds.DataflowJobId), + }}.createXYChartTile(), TileInfo{Title: "Dataflow Workers Max Backlog Time Seconds", TimeSeriesQueries: map[string]string{"": fmt.Sprintf(dataflowBacklogTimeQuery, resourceIds.DataflowJobId)}}.createXYChartTile(), } return dataflowTiles @@ -107,7 +104,7 @@ func createShardDataflowMetrics(resourceIds MonitoringMetricsResources) []*dashb func createShardDatastreamMetrics(resourceIds MonitoringMetricsResources) []*dashboardpb.MosaicLayout_Tile { datastreamTiles := []*dashboardpb.MosaicLayout_Tile{ TileInfo{ - Title: "Datastream Total Latency", + Title: "Datastream Total Latency", TimeSeriesQueries: map[string]string{"p50 " + resourceIds.DatastreamId: fmt.Sprintf(datastreamTotalLatencyQuery, resourceIds.DatastreamId, "50"), "p90 " + resourceIds.DatastreamId: fmt.Sprintf(datastreamTotalLatencyQuery, resourceIds.DatastreamId, "90")}}.createXYChartTile(), TileInfo{Title: "Datastream Throughput", TimeSeriesQueries: map[string]string{resourceIds.DatastreamId: fmt.Sprintf(datastreamThroughputQuery, resourceIds.DatastreamId)}}.createXYChartTile(), TileInfo{Title: "Datastream Unsupported Events", TimeSeriesQueries: map[string]string{resourceIds.DatastreamId: fmt.Sprintf(datastreamUnsupportedEventsQuery, resourceIds.DatastreamId)}}.createXYChartTile(), @@ -147,8 +144,8 @@ func createShardIndependentTopMetrics(resourceIds MonitoringMetricsResources) [] TileInfo{Title: "Datastream Unsupported Events", TimeSeriesQueries: map[string]string{resourceIds.DatastreamId: fmt.Sprintf(datastreamUnsupportedEventsQuery, resourceIds.DatastreamId)}}.createXYChartTile(), TileInfo{Title: "Pubsub Age of Oldest Unacknowledged Message", TimeSeriesQueries: map[string]string{resourceIds.PubsubSubscriptionId: fmt.Sprintf(pubsubOldestUnackedMessageAgeQuery, resourceIds.PubsubSubscriptionId)}}.createXYChartTile(), } - spannerMetrics:=createSpannerMetrics(resourceIds) - independentTopMetricsTiles=append(independentTopMetricsTiles,spannerMetrics...) + spannerMetrics := createSpannerMetrics(resourceIds) + independentTopMetricsTiles = append(independentTopMetricsTiles, spannerMetrics...) return independentTopMetricsTiles } @@ -178,12 +175,12 @@ func createAggDataflowMetrics(resourceIds MonitoringMetricsResources) []*dashboa "Max shard": fmt.Sprintf(dataflowAggCpuUtilMaxQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs)), }}.createXYChartTile(), TileInfo{ - Title: "Dataflow Workers Memory Utilization", + Title: "Dataflow Workers Memory Utilization", TimeSeriesQueries: map[string]string{ - "p50 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "50"), - "p90 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "90"), - "Max shard": fmt.Sprintf(dataflowAggMemoryUtilMaxQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs)), - }}.createXYChartTile(), + "p50 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "50"), + "p90 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "90"), + "Max shard": fmt.Sprintf(dataflowAggMemoryUtilMaxQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs)), + }}.createXYChartTile(), TileInfo{Title: "Dataflow Workers Max Backlog Time Seconds", TimeSeriesQueries: map[string]string{"Dataflow Backlog Time Seconds": fmt.Sprintf(dataflowAggBacklogTimeQuery, createAggFilterCondition("metric.job_id", dataflowJobs))}}.createXYChartTile(), TileInfo{Title: "Dataflow Per Shard Median CPU Utilization", TimeSeriesQueries: map[string]string{"": fmt.Sprintf(dataflowAggPerShardCpuUtil, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs))}}.createXYChartTile(), } @@ -197,7 +194,7 @@ func createAggDatastreamMetrics(resourceIds MonitoringMetricsResources) []*dashb } datastreamTiles := []*dashboardpb.MosaicLayout_Tile{ TileInfo{ - Title: "Datastream Total Latency", + Title: "Datastream Total Latency", TimeSeriesQueries: map[string]string{"p50 Datastream Latency": fmt.Sprintf(datastreamAggTotalLatencyQuery, createAggFilterCondition("resource.stream_id", datastreamJobs), "50", "50"), "p90 Datastream Latency": fmt.Sprintf(datastreamAggTotalLatencyQuery, createAggFilterCondition("resource.stream_id", datastreamJobs), "90", "90")}}.createXYChartTile(), TileInfo{Title: "Total Datastream Throughput", TimeSeriesQueries: map[string]string{"Datastream Total Throughput": fmt.Sprintf(datastreamAggThroughputQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(), TileInfo{Title: "Total Datastream Unsupported Events", TimeSeriesQueries: map[string]string{"Datastream Total Unsupported Events": fmt.Sprintf(datastreamAggUnsupportedEventsQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(), @@ -252,9 +249,9 @@ func createAggIndependentTopMetrics(resourceIds MonitoringMetricsResources) []*d TileInfo{Title: "Total Datastream Throughput", TimeSeriesQueries: map[string]string{"Datastream Throughput": fmt.Sprintf(datastreamAggThroughputQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(), TileInfo{Title: "Total Datastream Unsupported Events", TimeSeriesQueries: map[string]string{"Datastream Unsupported Events": fmt.Sprintf(datastreamAggUnsupportedEventsQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(), TileInfo{Title: "Pubsub Age of Oldest Unacknowledged Message", TimeSeriesQueries: map[string]string{"Pubsub Age of Oldest Unacknowledged Message": fmt.Sprintf(pubsubAggOldestUnackedMessageAgeQuery, createAggFilterCondition("resource.subscription_id", pubsubSubs))}}.createXYChartTile(), - } - spannerMetrics:=createSpannerMetrics(resourceIds) - independentTopMetricsTiles=append(independentTopMetricsTiles,spannerMetrics...) + } + spannerMetrics := createSpannerMetrics(resourceIds) + independentTopMetricsTiles = append(independentTopMetricsTiles, spannerMetrics...) return independentTopMetricsTiles } @@ -263,7 +260,7 @@ func createAggIndependentBottomMetrics(resourceIds MonitoringMetricsResources) [ for shardId, shardResource := range resourceIds.ShardToShardResourcesMap { shardUrl := fmt.Sprintf("https://console.cloud.google.com/monitoring/dashboards/builder/%v?project=%v", shardResource.MonitoringResources.DashboardName, resourceIds.ProjectId) shardString := fmt.Sprintf("Shard [%s](%s)", shardId, shardUrl) - if(shardToDashboardMappingText == ""){ + if shardToDashboardMappingText == "" { shardToDashboardMappingText = shardString } else { shardToDashboardMappingText += " \\\n" + shardString @@ -271,7 +268,7 @@ func createAggIndependentBottomMetrics(resourceIds MonitoringMetricsResources) [ } independentBottomMetricsTiles := []*dashboardpb.MosaicLayout_Tile{ TileInfo{ - Title: "Shard Dashboards", + Title: "Shard Dashboards", TextContent: shardToDashboardMappingText, }.createTextTile(), } @@ -332,14 +329,14 @@ func (tileInfo TileInfo) createCollapsibleGroupTile(tiles []*dashboardpb.MosaicL return &groupTile, heightOffset + groupTileHeight } -func (tileInfo TileInfo) createTextTile() (*dashboardpb.MosaicLayout_Tile){ - textTile := dashboardpb.MosaicLayout_Tile{ +func (tileInfo TileInfo) createTextTile() *dashboardpb.MosaicLayout_Tile { + textTile := dashboardpb.MosaicLayout_Tile{ Widget: &dashboardpb.Widget{ Title: tileInfo.Title, Content: &dashboardpb.Widget_Text{ Text: &dashboardpb.Text{ Content: tileInfo.TextContent, - Format: dashboardpb.Text_MARKDOWN, + Format: dashboardpb.Text_MARKDOWN, }, }, }, @@ -382,7 +379,7 @@ func getCreateMonitoringDashboardRequest( } // create bottom independent metrics tiles - if createAggIndependentBottomMetrics!= nil{ + if createAggIndependentBottomMetrics != nil { independentBottomMetricsTiles := createAggIndependentBottomMetrics(resourceIds) heightOffset += setWidgetPositions(independentBottomMetricsTiles, heightOffset) mosaicLayoutTiles = append(mosaicLayoutTiles, independentBottomMetricsTiles...) diff --git a/common/metrics/queries.go b/common/metrics/queries.go index f5f124d44e..4c7dd6ea39 100644 --- a/common/metrics/queries.go +++ b/common/metrics/queries.go @@ -4,16 +4,13 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -// Package utils contains common helper functions used across multiple other packages. -// Utils should not import any Spanner migration tool packages. package metrics // Defines queries for Monitoring Dashboard Metrics @@ -90,8 +87,8 @@ const ( "filter && (%s) | group_by 1m, [value_estimated_backlog_processing_time_mean: " + "mean(value.estimated_backlog_processing_time)] | every 1m | group_by [], [value_estimated_backlog_processing_time_mean_mean: " + "mean(value_estimated_backlog_processing_time_mean)]" - dataflowAggPerShardCpuUtil = "fetch gce_instance | metric 'compute.googleapis.com/instance/cpu/utilization' | filter (%s) " + - "| group_by 1m, [value_utilization_mean: mean(value.utilization)] | every 1m | group_by [metadata.user_labels.dataflow_job_id]," + + dataflowAggPerShardCpuUtil = "fetch gce_instance | metric 'compute.googleapis.com/instance/cpu/utilization' | filter (%s) " + + "| group_by 1m, [value_utilization_mean: mean(value.utilization)] | every 1m | group_by [metadata.user_labels.dataflow_job_id]," + " [value_utilization_mean_percentile: percentile(value_utilization_mean, 50)]" datastreamAggThroughputQuery = "fetch datastream.googleapis.com/Stream | metric 'datastream.googleapis.com/stream/event_count' | " + "filter (%s) | align rate(1m) | every 1m | group_by [], [value_event_count_aggregate: aggregate(value.event_count)]" diff --git a/common/utils/storage_utils.go b/common/utils/storage_utils.go new file mode 100644 index 0000000000..3968b1731e --- /dev/null +++ b/common/utils/storage_utils.go @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package utils contains common helper functions used across multiple other packages. +Utils should not import any Spanner migration tool packages. +*/ +package utils + +import ( + "fmt" + "net/url" + + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" +) + +func ParseGCSFilePath(filePath string) (*url.URL, error) { + if len(filePath) == 0 { + return nil, fmt.Errorf("found empty GCS path") + } + if filePath[len(filePath)-1] != '/' { + filePath = filePath + "/" + } + u, err := url.Parse(filePath) + if err != nil { + return nil, fmt.Errorf("parseFilePath: unable to parse file path %s", filePath) + } + if u.Scheme != constants.GCS_SCHEME { + return nil, fmt.Errorf("not a valid GCS path: %s, should start with 'gs'", filePath) + } + return u, nil +} diff --git a/common/utils/storage_utils_test.go b/common/utils/storage_utils_test.go new file mode 100644 index 0000000000..c9ea7f75a5 --- /dev/null +++ b/common/utils/storage_utils_test.go @@ -0,0 +1,87 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package utils + +import ( + "net/url" + "os" + "testing" + + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func TestParseGCSFilePath(t *testing.T) { + testCases := []struct { + name string + filePath string + expectError bool + want *url.URL + }{ + { + name: "Basic", + filePath: "gs://test-bucket/path/to/folder/", + expectError: false, + want: &url.URL{ + Scheme: "gs", + Host: "test-bucket", + Path: "/path/to/folder/", + }, + }, + { + name: "Append Slash", + filePath: "gs://test-bucket/path/to/folder", + expectError: false, + want: &url.URL{ + Scheme: "gs", + Host: "test-bucket", + Path: "/path/to/folder/", + }, + }, + { + name: "Empty File path", + filePath: "", + expectError: true, + want: nil, + }, + { + name: "Wrong Scheme", + filePath: "ab://testpath", + expectError: true, + want: nil, + }, + { + name: "Malformed Path", + filePath: "://path", + expectError: true, + want: nil, + }, + } + + for _, tc := range testCases { + got, err := ParseGCSFilePath(tc.filePath) + assert.Equal(t, tc.expectError, err != nil) + assert.Equal(t, tc.want, got) + } +} diff --git a/common/utils/utils.go b/common/utils/utils.go index 70ec53f882..6fc2569890 100644 --- a/common/utils/utils.go +++ b/common/utils/utils.go @@ -19,11 +19,11 @@ package utils import ( "bufio" "context" + "crypto/rand" "fmt" "io" "io/ioutil" "log" - "math/rand" "net/url" "os" "os/exec" @@ -37,6 +37,7 @@ import ( sp "cloud.google.com/go/spanner" database "cloud.google.com/go/spanner/admin/database/apiv1" instance "cloud.google.com/go/spanner/admin/instance/apiv1" + "cloud.google.com/go/spanner/admin/instance/apiv1/instancepb" "cloud.google.com/go/storage" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/internal" @@ -44,10 +45,8 @@ import ( "github.com/GoogleCloudPlatform/spanner-migration-tool/sources/spanner" "github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl" "golang.org/x/crypto/ssh/terminal" - "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "google.golang.org/api/option" - instancepb "google.golang.org/genproto/googleapis/spanner/admin/instance/v1" ) // IOStreams is a struct that contains the file descriptor for dumpFile. @@ -173,82 +172,6 @@ func PreloadGCSFiles(tables []ManifestTable) ([]ManifestTable, error) { return tables, nil } -func ParseGCSFilePath(filePath string) (*url.URL, error) { - if len(filePath) == 0 { - return nil, fmt.Errorf("found empty GCS path") - } - if filePath[len(filePath)-1] != '/' { - filePath = filePath + "/" - } - u, err := url.Parse(filePath) - if err != nil { - return nil, fmt.Errorf("parseFilePath: unable to parse file path %s", filePath) - } - if u.Scheme != constants.GCS_SCHEME { - return nil, fmt.Errorf("not a valid GCS path: %s, should start with 'gs'", filePath) - } - return u, nil -} - -func WriteToGCS(filePath, fileName, data string) error { - ctx := context.Background() - - client, err := storage.NewClient(ctx) - if err != nil { - fmt.Printf("Failed to create GCS client") - return err - } - defer client.Close() - u, err := ParseGCSFilePath(filePath) - if err != nil { - return fmt.Errorf("parseFilePath: unable to parse file path: %v", err) - } - bucketName := u.Host - bucket := client.Bucket(bucketName) - obj := bucket.Object(u.Path[1:] + fileName) - - w := obj.NewWriter(ctx) - if _, err := fmt.Fprint(w, data); err != nil { - fmt.Printf("Failed to write to Cloud Storage: %s", filePath) - return err - } - if err := w.Close(); err != nil { - fmt.Printf("Failed to close GCS file: %s", filePath) - return err - } - return nil -} - -func CreateGCSBucket(bucketName, projectID, location string) error { - ctx := context.Background() - - client, err := storage.NewClient(ctx) - if err != nil { - return fmt.Errorf("failed to create GCS client: %v", err) - } - defer client.Close() - bucket := client.Bucket(bucketName) - attrs := storage.BucketAttrs{ - Location: location, - } - if err := bucket.Create(ctx, projectID, &attrs); err != nil { - if e, ok := err.(*googleapi.Error); ok { - // Ignoring the bucket already exists error. - if e.Code != 409 { - return fmt.Errorf("failed to create bucket: %v", err) - } else { - fmt.Printf("Using the existing bucket: %v \n", bucketName) - } - } else { - return fmt.Errorf("failed to create bucket: %v", err) - } - - } else { - fmt.Printf("Created new GCS bucket: %v\n", bucketName) - } - return nil -} - // GetProject returns the cloud project we should use for accessing Spanner. // Use environment variable GCLOUD_PROJECT if it is set. // Otherwise, use the default project returned from gcloud. @@ -347,6 +270,12 @@ func GenerateName(prefix string) (string, error) { return fmt.Sprintf("%s_%x-%x", prefix, b[0:2], b[2:4]), nil } +func GenerateHashStr() string { + b := make([]byte, 4) + rand.Read(b) + return fmt.Sprintf("%x-%x", b[0:2], b[2:4]) +} + // parseURI parses an unknown URI string that could be a database, instance or project URI. func parseURI(URI string) (project, instance, dbName string) { project, instance, dbName = "", "", "" @@ -651,7 +580,7 @@ func CompareSchema(sessionFileConv, actualSpannerConv *internal.Conv) error { } else { if sessionColDef.Name != spannerColDef.Name || sessionColDef.T.IsArray != spannerColDef.T.IsArray || sessionColDef.T.Len != spannerColDef.T.Len || sessionColDef.T.Name != spannerColDef.T.Name || sessionColDef.NotNull != spannerColDef.NotNull { - return fmt.Errorf("column detail for table %v don't match: session column: %v, spanner column: %v", sessionTable.Name, sessionColDef, spannerColDef) + return fmt.Errorf("column detail for table %v don't match: session column: %v, spanner column: %v", sessionTable.Name, sessionColDef, spannerColDef) } } } @@ -708,3 +637,12 @@ func FindInPrimaryKey(id string, primaryKeys []ddl.IndexKey) bool { } return false } + +func Contains[S ~[]E, E comparable](s S, v E) bool { + for i := range s { + if v == s[i] { + return true + } + } + return false +} diff --git a/conversion/conversion.go b/conversion/conversion.go index 20216dfea4..bf02976781 100644 --- a/conversion/conversion.go +++ b/conversion/conversion.go @@ -44,6 +44,8 @@ import ( datastream "cloud.google.com/go/datastream/apiv1" sp "cloud.google.com/go/spanner" database "cloud.google.com/go/spanner/admin/database/apiv1" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/metrics" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" @@ -67,12 +69,12 @@ import ( dydb "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodbstreams" mysqldriver "github.com/go-sql-driver/mysql" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" "go.uber.org/zap" adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/proto" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/stdlib" ) var ( @@ -354,8 +356,9 @@ func dataFromDatabase(ctx context.Context, sourceProfile profiles.SourceProfile, // Try to apply lifecycle rule to Datastream destination bucket. gcsConfig := streamingCfg.GcsCfg + sa := storageaccessor.StorageAccessorImpl{} if gcsConfig.TtlInDaysSet { - err = streaming.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays) + err = sa.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays) if err != nil { logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err)) logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n") @@ -475,8 +478,9 @@ func dataFromDatabaseForDataflowMigration(targetProfile profiles.TargetProfile, // Try to apply lifecycle rule to Datastream destination bucket. gcsConfig := streamingCfg.GcsCfg + sa := storageaccessor.StorageAccessorImpl{} if gcsConfig.TtlInDaysSet { - err = streaming.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays) + err = sa.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays) if err != nil { logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err)) logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n") @@ -520,11 +524,11 @@ func dataFromDatabaseForDataflowMigration(targetProfile profiles.TargetProfile, // create monitoring aggregated dashboard for sharded migration aggMonitoringResources := metrics.MonitoringMetricsResources{ - ProjectId: targetProfile.Conn.Sp.Project, - SpannerInstanceId: targetProfile.Conn.Sp.Instance, - SpannerDatabaseId: targetProfile.Conn.Sp.Dbname, - ShardToShardResourcesMap: conv.Audit.StreamingStats.ShardToShardResourcesMap, - MigrationRequestId: conv.Audit.MigrationRequestId, + ProjectId: targetProfile.Conn.Sp.Project, + SpannerInstanceId: targetProfile.Conn.Sp.Instance, + SpannerDatabaseId: targetProfile.Conn.Sp.Dbname, + ShardToShardResourcesMap: conv.Audit.StreamingStats.ShardToShardResourcesMap, + MigrationRequestId: conv.Audit.MigrationRequestId, } aggRespDash, dashboardErr := aggMonitoringResources.CreateDataflowAggMonitoringDashboard(ctx) if dashboardErr != nil { @@ -798,7 +802,8 @@ func getSeekable(f *os.File) (*os.File, int64, error) { // VerifyDb checks whether the db exists and if it does, verifies if the schema is what we currently support. func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (dbExists bool, err error) { - dbExists, err = CheckExistingDb(ctx, adminClient, dbURI) + spA := spanneraccessor.SpannerAccessorImpl{} + dbExists, err = spA.CheckExistingDb(ctx, dbURI) if err != nil { return dbExists, err } @@ -808,31 +813,6 @@ func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, db return dbExists, err } -// CheckExistingDb checks whether the database with dbURI exists or not. -// If API call doesn't respond then user is informed after every 5 minutes on command line. -func CheckExistingDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (bool, error) { - gotResponse := make(chan bool) - var err error - go func() { - _, err = adminClient.GetDatabase(ctx, &adminpb.GetDatabaseRequest{Name: dbURI}) - gotResponse <- true - }() - for { - select { - case <-time.After(5 * time.Minute): - fmt.Println("WARNING! API call not responding: make sure that spanner api endpoint is configured properly") - case <-gotResponse: - if err != nil { - if utils.ContainsAny(strings.ToLower(err.Error()), []string{"database not found"}) { - return false, nil - } - return false, fmt.Errorf("can't get database info: %s", err) - } - return true, nil - } - } -} - // ValidateTables validates that all the tables in the database are empty. // It returns the name of the first non-empty table if found, and an empty string otherwise. func ValidateTables(ctx context.Context, client *sp.Client, spDialect string) (string, error) { @@ -1305,20 +1285,20 @@ func GetInfoSchemaFromCloudSQL(sourceProfile profiles.SourceProfile, targetProfi switch driver { case constants.MYSQL: d, err := cloudsqlconn.NewDialer(context.Background(), cloudsqlconn.WithIAMAuthN()) - if err != nil { - return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err) - } - var opts []cloudsqlconn.DialOption + if err != nil { + return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err) + } + var opts []cloudsqlconn.DialOption instanceName := fmt.Sprintf("%s:%s:%s", sourceProfile.ConnCloudSQL.Mysql.Project, sourceProfile.ConnCloudSQL.Mysql.Region, sourceProfile.ConnCloudSQL.Mysql.InstanceName) - mysqldriver.RegisterDialContext("cloudsqlconn", - func(ctx context.Context, addr string) (net.Conn, error) { - return d.Dial(ctx, instanceName, opts...) - }) + mysqldriver.RegisterDialContext("cloudsqlconn", + func(ctx context.Context, addr string) (net.Conn, error) { + return d.Dial(ctx, instanceName, opts...) + }) - dbURI := fmt.Sprintf("%s:empty@cloudsqlconn(localhost:3306)/%s?parseTime=true", - sourceProfile.ConnCloudSQL.Mysql.User, sourceProfile.ConnCloudSQL.Mysql.Db) + dbURI := fmt.Sprintf("%s:empty@cloudsqlconn(localhost:3306)/%s?parseTime=true", + sourceProfile.ConnCloudSQL.Mysql.User, sourceProfile.ConnCloudSQL.Mysql.Db) - db, err := sql.Open("mysql", dbURI) + db, err := sql.Open("mysql", dbURI) if err != nil { return nil, fmt.Errorf("sql.Open: %w", err) } @@ -1330,25 +1310,25 @@ func GetInfoSchemaFromCloudSQL(sourceProfile profiles.SourceProfile, targetProfi }, nil case constants.POSTGRES: d, err := cloudsqlconn.NewDialer(context.Background(), cloudsqlconn.WithIAMAuthN()) - if err != nil { - return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err) - } - var opts []cloudsqlconn.DialOption - - dsn := fmt.Sprintf("user=%s database=%s", sourceProfile.ConnCloudSQL.Pg.User, sourceProfile.ConnCloudSQL.Pg.Db) - config, err := pgx.ParseConfig(dsn) - if err != nil { - return nil, err - } + if err != nil { + return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err) + } + var opts []cloudsqlconn.DialOption + + dsn := fmt.Sprintf("user=%s database=%s", sourceProfile.ConnCloudSQL.Pg.User, sourceProfile.ConnCloudSQL.Pg.Db) + config, err := pgx.ParseConfig(dsn) + if err != nil { + return nil, err + } instanceName := fmt.Sprintf("%s:%s:%s", sourceProfile.ConnCloudSQL.Pg.Project, sourceProfile.ConnCloudSQL.Pg.Region, sourceProfile.ConnCloudSQL.Pg.InstanceName) - config.DialFunc = func(ctx context.Context, network, instance string) (net.Conn, error) { - return d.Dial(ctx, instanceName, opts...) - } - dbURI := stdlib.RegisterConnConfig(config) - db, err := sql.Open("pgx", dbURI) - if err != nil { - return nil, fmt.Errorf("sql.Open: %w", err) - } + config.DialFunc = func(ctx context.Context, network, instance string) (net.Conn, error) { + return d.Dial(ctx, instanceName, opts...) + } + dbURI := stdlib.RegisterConnConfig(config) + db, err := sql.Open("pgx", dbURI) + if err != nil { + return nil, fmt.Errorf("sql.Open: %w", err) + } temp := false return postgres.InfoSchemaImpl{ Db: db, diff --git a/dao/dao.go b/dao/dao.go new file mode 100644 index 0000000000..c515e39539 --- /dev/null +++ b/dao/dao.go @@ -0,0 +1,320 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dao + +import ( + "context" + "fmt" + + "cloud.google.com/go/spanner" + "google.golang.org/api/iterator" +) + +type StateData struct { + State string `json:"state"` +} + +type DAO interface { + InsertSMTJobEntry(ctx context.Context, jobId, jobName, jobType, dialect, dbName string, jobData spanner.NullJSON) error + UpdateSMTJobState(ctx context.Context, jobId, state string) error + InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId, resourceName, resourceType string, resourceData spanner.NullJSON) error + UpdateSMTResourceState(ctx context.Context, resourceId, state string) error + UpdateSMTResourceExternalId(ctx context.Context, resourceId, externalId string) error +} + +type DAOImpl struct{} + +// Insert a job entry into the SMT_JOB table. +func (dao *DAOImpl) InsertSMTJobEntry(ctx context.Context, jobId, jobName, jobType, dialect, dbName string, jobData spanner.NullJSON) error { + _, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + jobStmt := spanner.Statement{ + SQL: `INSERT INTO SMT_JOB + (JobId, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt, UpdatedAt) + VALUES( + @jobId, @jobName, @jobType, @jobStateData, @jobData, @dialect, @dbName, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP() + );`, + Params: map[string]interface{}{ + "jobId": jobId, + "jobName": jobName, + "jobType": jobType, + "jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}}, + "jobData": jobData, + "dialect": dialect, + "dbName": dbName, + }, + } + _, err := txn.Update(ctx, jobStmt) + if err != nil { + return err + } + // Update job history table within the same txn. + _, err = updateJobHistoryWithinTxn(ctx, txn, jobId) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("could not insert SMT job entry: %v", err) + } + return nil +} + +// Update the state of the SMT job. +func (dao *DAOImpl) UpdateSMTJobState(ctx context.Context, jobId, state string) error { + _, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + jobStmt := spanner.Statement{ + SQL: `UPDATE SMT_JOB SET JobStateData = @jobStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP() + WHERE JobId = @jobId;`, + Params: map[string]interface{}{ + "jobId": jobId, + "jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}}, + }, + } + _, err := txn.Update(ctx, jobStmt) + if err != nil { + return err + } + _, err = updateJobHistoryWithinTxn(ctx, txn, jobId) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("error updating smt job state: %v", err) + } + return nil +} + +// Insert an entry into the SMT_RESOURCE table. +func (dao *DAOImpl) InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId, resourceName, resourceType string, resourceData spanner.NullJSON) error { + _, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + resourceStmt := spanner.Statement{ + SQL: `INSERT INTO SMT_RESOURCE + (ResourceId, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt, UpdatedAt) + VALUES( + @resourceId, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP() + );`, + Params: map[string]interface{}{ + "resourceId": resourceId, + "jobId": jobId, + "externalId": externalId, + "resourceName": resourceName, + "resourceType": resourceType, + "resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}}, + "resourceData": resourceData, + }, + } + _, err := txn.Update(ctx, resourceStmt) + if err != nil { + return err + } + // Update the resource history table in the same transaction. + _, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("error inserting smt resource entry: %v", err) + } + return nil +} + +// Update the state of the SMT resource. +func (dao *DAOImpl) UpdateSMTResourceState(ctx context.Context, resourceId, state string) error { + _, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + jobStmt := spanner.Statement{ + SQL: `UPDATE SMT_RESOURCE SET ResourceStateData = @resourceStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP() + WHERE ResourceId = @resourceId;`, + Params: map[string]interface{}{ + "resourceId": resourceId, + "resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}}, + }, + } + _, err := txn.Update(ctx, jobStmt) + if err != nil { + return err + } + _, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("error updating smt resource state: %v", err) + } + return nil +} + +// Update the external of the SMT resource. +func (dao *DAOImpl) UpdateSMTResourceExternalId(ctx context.Context, resourceId, externalId string) error { + _, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + jobStmt := spanner.Statement{ + SQL: `UPDATE SMT_RESOURCE SET ExternalId = @externalId, UpdatedAt = PENDING_COMMIT_TIMESTAMP() + WHERE ResourceId = @resourceId;`, + Params: map[string]interface{}{ + "resourceId": resourceId, + "externalId": externalId, + }, + } + _, err := txn.Update(ctx, jobStmt) + if err != nil { + return err + } + _, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("error updating smt resource external id: %v", err) + } + return nil +} + +func updateJobHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) { + version, err := getLatestJobVersionWithinTxn(ctx, txn, jobId) + if err != nil { + return 0, fmt.Errorf("error fetching latest job version: %v", err) + } + // Fetch the newly updated row from SMT_JOB table. + stmt := spanner.Statement{SQL: ` + SELECT + JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName + FROM SMT_JOB WHERE JobId = @jobId;`, + Params: map[string]interface{}{"jobId": jobId}, + } + iter := txn.Query(ctx, stmt) + defer iter.Stop() + var jobName, jobType, dialect, spannerDatabaseName spanner.NullString + var jobStateData, jobData spanner.NullJSON + row, err := iter.Next() + if err == iterator.Done || err != nil { + return 0, err + } + if err := row.Columns(&jobName, &jobType, &jobStateData, &jobData, &dialect, &spannerDatabaseName); err != nil { + return 0, fmt.Errorf("error reading smt job row: %v", err) + } + + // Insert entry to SMT_JOB_HISTORY table. + jobStmt := spanner.Statement{ + SQL: `INSERT INTO SMT_JOB_HISTORY + (JobId, Version, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt) + VALUES( + @jobId, @version, @jobName, @jobType, @jobStateData, @jobData, @dialect, @spannerDatabaseName, PENDING_COMMIT_TIMESTAMP() + );`, + Params: map[string]interface{}{ + "jobId": jobId, + "version": version + 1, + "jobName": jobName, + "jobType": jobType, + "jobStateData": jobStateData, + "jobData": jobData, + "dialect": dialect, + "spannerDatabaseName": spannerDatabaseName, + }, + } + return txn.Update(ctx, jobStmt) +} + +func getLatestJobVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) { + // Fetch latest version for the job from history table. + stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_JOB_HISTORY WHERE JobId = @jobId;`, + Params: map[string]interface{}{"jobId": jobId}, + } + iter := txn.Query(ctx, stmt) + defer iter.Stop() + version := spanner.NullInt64{} + row, err := iter.Next() + if err == iterator.Done || err != nil { + return 0, err + } + if err := row.Columns(&version); err != nil { + return 0, err + } + if version.Valid { + return version.Int64, nil + } + return 0, nil +} + +func updateResourceHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) { + version, err := getLatestResourceVersionWithinTxn(ctx, txn, resourceId) + if err != nil { + return 0, fmt.Errorf("error fetching latest resource version: %v", err) + } + // Fetch the newly updated row from SMT_RESOURCE table. + stmt := spanner.Statement{SQL: ` + SELECT + JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData + FROM SMT_RESOURCE WHERE ResourceId = @resourceId;`, + Params: map[string]interface{}{"resourceId": resourceId}, + } + iter := txn.Query(ctx, stmt) + defer iter.Stop() + var jobId, externalId, resourceName, resourceType spanner.NullString + var resourceStateData, resourceData spanner.NullJSON + row, err := iter.Next() + if err == iterator.Done || err != nil { + return 0, err + } + if err := row.Columns(&jobId, &externalId, &resourceName, &resourceType, &resourceStateData, &resourceData); err != nil { + return 0, fmt.Errorf("error reading smt resource row: %v", err) + } + // Create new entry into the SMT_RESOURCE_HISTORY table. + jobStmt := spanner.Statement{ + SQL: `INSERT INTO SMT_RESOURCE_HISTORY + (ResourceId, Version, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt) + VALUES( + @resourceId, @version, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP() + );`, + Params: map[string]interface{}{ + "resourceId": resourceId, + "version": version + 1, + "jobId": jobId, + "externalId": externalId, + "resourceName": resourceName, + "resourceType": resourceType, + "resourceStateData": resourceStateData, + "resourceData": resourceData, + }, + } + return txn.Update(ctx, jobStmt) +} + +func getLatestResourceVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) { + // Fetch latest version for the resource from history table. + stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_RESOURCE_HISTORY WHERE ResourceId = @resourceId;`, + Params: map[string]interface{}{"resourceId": resourceId}, + } + iter := txn.Query(ctx, stmt) + defer iter.Stop() + version := spanner.NullInt64{} + row, err := iter.Next() + if err == iterator.Done || err != nil { + return 0, err + } + if err := row.Columns(&version); err != nil { + return 0, err + } + if version.Valid { + return version.Int64, nil + } + return 0, nil +} diff --git a/dao/dao_client.go b/dao/dao_client.go new file mode 100644 index 0000000000..46964888de --- /dev/null +++ b/dao/dao_client.go @@ -0,0 +1,48 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dao + +import ( + "context" + "fmt" + "sync" + + sp "cloud.google.com/go/spanner" +) + +var once sync.Once +var spClient *sp.Client + +// This function is declared as a global variable to make it testable. The unit +// tests edit this function, acting like a double. +var newClient = sp.NewClient + +func GetOrCreateClient(ctx context.Context, dbURI string) (*sp.Client, error) { + var err error + if spClient == nil { + once.Do(func() { + spClient, err = newClient(ctx, dbURI) + }) + if err != nil { + return nil, fmt.Errorf("failed to create spanner database client: %v", err) + } + return spClient, nil + } + return spClient, nil +} + +// The DAO client must be initiated via GetOrCreateClient() once before using GetClient(). +func GetClient() *sp.Client { + return spClient +} diff --git a/dao/dao_client_test.go b/dao/dao_client_test.go new file mode 100644 index 0000000000..05c0352a9a --- /dev/null +++ b/dao/dao_client_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dao + +import ( + "context" + "fmt" + "os" + "sync" + "testing" + + sp "cloud.google.com/go/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "google.golang.org/api/option" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +func resetTest() { + spClient = nil + once = sync.Once{} +} + +func TestGetOrCreateClient_Basic(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return &sp.Client{}, nil + } + c, err := GetOrCreateClient(ctx, "testURI") + assert.NotNil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return &sp.Client{}, nil + } + c, err := GetOrCreateClient(ctx, "testURI") + assert.NotNil(t, c) + assert.Nil(t, err) + // Explicitly set the client to nil. Running GetOrCreateClient should not create a + // new client since sync would already be executed. + spClient = nil + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return nil, fmt.Errorf("test error") + } + c, err = GetOrCreateClient(ctx, "testURI") + assert.Nil(t, c) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return &sp.Client{}, nil + } + oldC, err := GetOrCreateClient(ctx, "testURI") + assert.NotNil(t, oldC) + assert.Nil(t, err) + + // Explicitly reset once. Running GetOrCreateClient should not create a + // new client the if condition should prevent it. + once = sync.Once{} + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return nil, fmt.Errorf("test error") + } + newC, err := GetOrCreateClient(ctx, "testURI") + assert.Equal(t, oldC, newC) + assert.Nil(t, err) +} + +func TestGetOrCreateClient_Error(t *testing.T) { + resetTest() + ctx := context.Background() + oldFunc := newClient + defer func() { newClient = oldFunc }() + + newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) { + return nil, fmt.Errorf("test error") + } + c, err := GetOrCreateClient(ctx, "testURI") + assert.Nil(t, c) + assert.NotNil(t, err) +} diff --git a/reverserepl/activity/IActivity.go b/reverserepl/activity/IActivity.go new file mode 100644 index 0000000000..245ec2e8e1 --- /dev/null +++ b/reverserepl/activity/IActivity.go @@ -0,0 +1,21 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import "context" + +type Activity interface { + Transaction(ctx context.Context) error + Compensation(ctx context.Context) error +} diff --git a/reverserepl/activity/create_smt_job_entry.go b/reverserepl/activity/create_smt_job_entry.go new file mode 100644 index 0000000000..9ef0d60827 --- /dev/null +++ b/reverserepl/activity/create_smt_job_entry.go @@ -0,0 +1,61 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + + "cloud.google.com/go/spanner" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/dao" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" +) + +type CreateSmtJobEntryInput struct { + SmtJobId string + JobName string + SpannerProjectId string + InstanceId string + DatabaseId string + JobData string +} + +type CreateSmtJobEntry struct { + Input *CreateSmtJobEntryInput + DAO dao.DAO + SpA spanneraccessor.SpannerAccessor +} + +// This creates a reverse replication entry in the SMT job table. +func (p *CreateSmtJobEntry) Transaction(ctx context.Context) error { + input := p.Input + dialect, err := p.SpA.GetDatabaseDialect(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", input.SpannerProjectId, input.InstanceId, input.DatabaseId)) + if err != nil { + return fmt.Errorf("could not fetch database dialect: %v", err) + } + logger.Log.Debug(fmt.Sprintf("found database dialect: %s", dialect)) + jobData := spanner.NullJSON{Valid: true, Value: input.JobData} + err = p.DAO.InsertSMTJobEntry(ctx, input.SmtJobId, input.JobName, constants.REVERSE_REPLICATION_JOB_TYPE, dialect, input.DatabaseId, jobData) + if err != nil { + return err + } + logger.Log.Debug("Created entry SMT Job entry") + return nil +} + +func (p *CreateSmtJobEntry) Compensation(ctx context.Context) error { + return nil +} diff --git a/reverserepl/activity/create_smt_job_entry_test.go b/reverserepl/activity/create_smt_job_entry_test.go new file mode 100644 index 0000000000..e28e43c264 --- /dev/null +++ b/reverserepl/activity/create_smt_job_entry_test.go @@ -0,0 +1,109 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + "os" + "testing" + + "cloud.google.com/go/spanner" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/dao" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func init() { + logger.Log = zap.NewNop() +} + +func TestMain(m *testing.M) { + res := m.Run() + os.Exit(res) +} + +type SpannerAccessorMock struct { + spanneraccessor.SpannerAccessor +} + +var getDatabaseDialectMock func(ctx context.Context, dbURI string) (string, error) + +func (sam *SpannerAccessorMock) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) { + return getDatabaseDialectMock(ctx, dbURI) +} + +type DAOMock struct { + dao.DAOImpl +} + +var insertSMTJobEntryMock func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error + +func (dao *DAOMock) InsertSMTJobEntry(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error { + return insertSMTJobEntryMock(ctx, jobId, jobName, jobType, dialect, dbName, jobData) +} + +func TestCreateSmtJobEntryTransaction(t *testing.T) { + testCases := []struct { + name string + getDatabaseDialectMock func(ctx context.Context, dbURI string) (string, error) + insertSMTJobEntryMock func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error + expectError bool + }{ + { + name: "No errors", + getDatabaseDialectMock: func(ctx context.Context, dbURI string) (string, error) { + return "", nil + }, + insertSMTJobEntryMock: func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error { + return nil + }, + expectError: false, + }, + { + name: "Fetch Dialect error", + getDatabaseDialectMock: func(ctx context.Context, dbURI string) (string, error) { + return "", fmt.Errorf("test error") + }, + insertSMTJobEntryMock: func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error { + return nil + }, + expectError: true, + }, + { + name: "Dao error", + getDatabaseDialectMock: func(ctx context.Context, dbURI string) (string, error) { + return "", nil + }, + insertSMTJobEntryMock: func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error { + return fmt.Errorf("test error") + }, + expectError: true, + }, + } + ctx := context.Background() + createSmtJobEntry := CreateSmtJobEntry{ + Input: &CreateSmtJobEntryInput{}, + DAO: &DAOMock{}, + SpA: &SpannerAccessorMock{}, + } + for _, tc := range testCases { + getDatabaseDialectMock = tc.getDatabaseDialectMock + insertSMTJobEntryMock = tc.insertSMTJobEntryMock + err := createSmtJobEntry.Transaction(ctx) + assert.Equal(t, tc.expectError, err != nil) + } +} diff --git a/reverserepl/activity/prepare_change_stream.go b/reverserepl/activity/prepare_change_stream.go new file mode 100644 index 0000000000..368fcc2606 --- /dev/null +++ b/reverserepl/activity/prepare_change_stream.go @@ -0,0 +1,71 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource" +) + +type PrepareChangeStreamInput struct { + SmtJobId string + ChangeStreamName string + DbURI string +} + +type PrepareChangeStreamOutput struct { + Exists bool + ExistsWithIncorrectOptions bool + Created bool +} + +type PrepareChangeStream struct { + Input *PrepareChangeStreamInput + Output *PrepareChangeStreamOutput + SpA spanneraccessor.SpannerAccessor +} + +// This checks is a valid change stream exists or not. If not, it creates one on the provided DbURI. +func (p *PrepareChangeStream) Transaction(ctx context.Context) error { + input := p.Input + csExists, err := p.SpA.CheckIfChangeStreamExists(ctx, input.ChangeStreamName, input.DbURI) + if err != nil { + return err + } + if csExists { + err = p.SpA.ValidateChangeStreamOptions(ctx, input.ChangeStreamName, input.DbURI) + if err != nil { + p.Output.ExistsWithIncorrectOptions = true + return fmt.Errorf("invalid change stream option found: %v", err) + } + logger.Log.Info(fmt.Sprintf("change stream %s already exists for %s, skipping creation", input.ChangeStreamName, input.DbURI)) + p.Output.Exists = true + return nil + } + err = resource.CreateChangeStreamSMTResource(ctx, p.SpA, input.SmtJobId, input.ChangeStreamName, input.DbURI) + if err != nil { + return fmt.Errorf("could not create change stream resource: %v", err) + } + logger.Log.Info(fmt.Sprintf("Created change stream %s for %s", input.ChangeStreamName, input.DbURI)) + p.Output.Created = true + return nil +} + +func (p *PrepareChangeStream) Compensation(ctx context.Context) error { + return nil +} diff --git a/reverserepl/activity/prepare_dataflow_reader.go b/reverserepl/activity/prepare_dataflow_reader.go new file mode 100644 index 0000000000..70880952c1 --- /dev/null +++ b/reverserepl/activity/prepare_dataflow_reader.go @@ -0,0 +1,146 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + "slices" + + dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource" +) + +type PrepareDataflowReaderInput struct { + SmtJobId string + ChangeStreamName string + InstanceId string + DatabaseId string + SpannerProjectId string + SessionFilePath string + SourceShardsFilePath string + MetadataInstance string + MetadataDatabase string + GcsOutputDirectory string + StartTimestamp string + EndTimestamp string + WindowDuration string + FiltrationMode string + MetadataTableSuffix string + SkipDirectoryName string + ShardingCustomJarPath string + ShardingCustomClassName string + TuningCfg string + SpannerLocation string +} + +type PrepareDataflowReaderOutput struct { + JobId string +} + +type PrepareDataflowReader struct { + Input *PrepareDataflowReaderInput + Output *PrepareDataflowReaderOutput + DfA dataflowaccessor.DataflowAccessor + SA storageaccessor.StorageAccessor +} + +// Launches the reader dataflow job. +func (p *PrepareDataflowReader) Transaction(ctx context.Context) error { + input := p.Input + readerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, p.SA, input.TuningCfg) + if err != nil { + return fmt.Errorf("error reading reader tuning config %s: %v", input.TuningCfg, err) + } + logger.Log.Debug(fmt.Sprintf("readerTuningCfg: %+v", readerTuningCfg)) + validateUpdateReaderTuningCfg(&readerTuningCfg, input.SpannerProjectId, input.SpannerLocation, input.SmtJobId) + logger.Log.Debug(fmt.Sprintf("Updated readerTuningCfg: %+v", readerTuningCfg)) + + params := map[string]string{ + "changeStreamName": input.ChangeStreamName, + "instanceId": input.InstanceId, + "databaseId": input.DatabaseId, + "spannerProjectId": input.SpannerProjectId, + "metadataInstance": input.MetadataInstance, + "metadataDatabase": input.MetadataDatabase, + "gcsOutputDirectory": input.GcsOutputDirectory, + "sessionFilePath": input.SessionFilePath, + "sourceShardsFilePath": input.SourceShardsFilePath, + "endTimestamp": input.EndTimestamp, + "windowDuration": input.WindowDuration, + "filtrationMode": input.FiltrationMode, + "metadataTableSuffix": input.MetadataTableSuffix, + "skipDirectoryName": input.SkipDirectoryName, + "startTimestamp": input.StartTimestamp, + "runIdentifier": input.SmtJobId, + "runMode": constants.RR_READER_REGULAR_MODE, + } + // Cannot send empty strings since the template expects GCS file paths. + if input.ShardingCustomJarPath != "" { + params["shardingCustomJarPath"] = input.ShardingCustomJarPath + params["shardingCustomClassName"] = input.ShardingCustomClassName + } + dfLaunchReq, err := dataflowutils.GetDataflowLaunchRequest(params, readerTuningCfg) + if err != nil { + return err + } + dfJobId, err := resource.CreateDataflowSMTResource(ctx, p.DfA, input.SmtJobId, dfLaunchReq) + if err != nil { + return err + } + logger.Log.Info(fmt.Sprintf("Launched reader job with id: %s", dfJobId)) + logger.Log.Info(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", dfLaunchReq.LaunchParameter.JobName, dataflowutils.GetGcloudDataflowCommand(dfLaunchReq))) + p.Output.JobId = dfJobId + return nil +} + +func (p *PrepareDataflowReader) Compensation(ctx context.Context) error { + return nil +} + +func validateUpdateReaderTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, spannerProjectId, spannerLocation, smtJobId string) { + if cfg.ProjectId == "" { + cfg.ProjectId = spannerProjectId + } + if cfg.JobName == "" { + cfg.JobName = fmt.Sprintf("smt-reverse-replication-reader-%s", utils.GenerateHashStr()) + } + if cfg.Location == "" { + cfg.Location = spannerLocation + } + if cfg.MaxWorkers == 0 { + cfg.MaxWorkers = 50 + } + if cfg.NumWorkers == 0 { + cfg.NumWorkers = 5 + } + if cfg.MachineType == "" { + cfg.MachineType = "n1-standard-2" + } + cfg.AdditionalUserLabels["smt-reverse-replication-reader"] = smtJobId + if cfg.GcsTemplatePath == "" { + cfg.GcsTemplatePath = constants.REVERSE_REPLICATION_READER_TEMPLATE_PATH + } + if cfg.AdditionalExperiments == nil { + cfg.AdditionalExperiments = []string{"use_runner_v2"} + } else if !slices.Contains(cfg.AdditionalExperiments, "use_runner_v2") { + cfg.AdditionalExperiments = append(cfg.AdditionalExperiments, "use_runner_v2") + } + cfg.EnableStreamingEngine = true +} diff --git a/reverserepl/activity/prepare_dataflow_writer.go b/reverserepl/activity/prepare_dataflow_writer.go new file mode 100644 index 0000000000..22c1bbe26f --- /dev/null +++ b/reverserepl/activity/prepare_dataflow_writer.go @@ -0,0 +1,125 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + + dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource" +) + +type PrepareDataflowWriterInput struct { + SmtJobId string + SourceShardsFilePath string + SessionFilePath string + SourceType string + SourceDbTimezoneOffset string + TimerInterval int + StartTimestamp string + WindowDuration string + GCSInputDirectoryPath string + SpannerProjectId string + MetadataInstance string + MetadataDatabase string + MetadataTableSuffix string + TuningCfg string + SpannerLocation string +} + +type PrepareDataflowWriterOutput struct { + JobId string +} + +type PrepareDataflowWriter struct { + Input *PrepareDataflowWriterInput + Output *PrepareDataflowWriterOutput + DfA dataflowaccessor.DataflowAccessor + SA storageaccessor.StorageAccessor +} + +// Launches the writer dataflow job. +func (p *PrepareDataflowWriter) Transaction(ctx context.Context) error { + input := p.Input + writerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, p.SA, input.TuningCfg) + if err != nil { + return fmt.Errorf("error reading writer tuning config %s: %v", input.TuningCfg, err) + } + logger.Log.Debug(fmt.Sprintf("writerTuningCfg: %+v", writerTuningCfg)) + validateUpdateWriterTuningCfg(&writerTuningCfg, input.SpannerProjectId, input.SpannerLocation, input.SmtJobId) + logger.Log.Debug(fmt.Sprintf("Updated writerTuningCfg: %+v", writerTuningCfg)) + params := map[string]string{ + "sourceShardsFilePath": input.SourceShardsFilePath, + "sessionFilePath": input.SessionFilePath, + "sourceType": input.SourceType, + "sourceDbTimezoneOffset": input.SourceDbTimezoneOffset, + "timerInterval": fmt.Sprintf("%v", input.TimerInterval), + "windowDuration": input.WindowDuration, + "GCSInputDirectoryPath": input.GCSInputDirectoryPath, + "metadataTableSuffix": input.MetadataTableSuffix, + "spannerProjectId": input.SpannerProjectId, + "metadataInstance": input.MetadataInstance, + "metadataDatabase": input.MetadataDatabase, + "startTimestamp": input.StartTimestamp, + "runIdentifier": input.SmtJobId, + "runMode": constants.RR_WRITER_REGULAR_MODE, + } + dfLaunchReq, err := dataflowutils.GetDataflowLaunchRequest(params, writerTuningCfg) + if err != nil { + return err + } + dfJobId, err := resource.CreateDataflowSMTResource(ctx, p.DfA, input.SmtJobId, dfLaunchReq) + if err != nil { + return err + } + logger.Log.Info(fmt.Sprintf("Launched writer job with id: %s", dfJobId)) + logger.Log.Info(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", dfLaunchReq.LaunchParameter.JobName, dataflowutils.GetGcloudDataflowCommand(dfLaunchReq))) + p.Output.JobId = dfJobId + return nil +} + +func (p *PrepareDataflowWriter) Compensation(ctx context.Context) error { + return nil +} + +func validateUpdateWriterTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, spannerProjectId, spannerLocation, smtJobId string) { + if cfg.ProjectId == "" { + cfg.ProjectId = spannerProjectId + } + if cfg.JobName == "" { + cfg.JobName = fmt.Sprintf("smt-reverse-replication-writer-%s", utils.GenerateHashStr()) + } + if cfg.Location == "" { + cfg.Location = spannerLocation + } + if cfg.MaxWorkers == 0 { + cfg.MaxWorkers = 50 + } + if cfg.NumWorkers == 0 { + cfg.NumWorkers = 5 + } + if cfg.MachineType == "" { + cfg.MachineType = "n1-standard-2" + } + cfg.AdditionalUserLabels["smt-reverse-replication-writer"] = smtJobId + if cfg.GcsTemplatePath == "" { + cfg.GcsTemplatePath = constants.REVERSE_REPLICATION_WRITER_TEMPLATE_PATH + } +} diff --git a/reverserepl/activity/prepare_gcs_bucket.go b/reverserepl/activity/prepare_gcs_bucket.go new file mode 100644 index 0000000000..24db80398f --- /dev/null +++ b/reverserepl/activity/prepare_gcs_bucket.go @@ -0,0 +1,76 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + "strings" + + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource" +) + +type PrepareGcsBucketInput struct { + SmtJobId string + SmtBucketName string + SpannerProjectId string + SpannerLocation string + SessionFilePath string + SourceConnectionConfig string + IsSMTBucketRequired bool +} + +type PrepareGcsBucketOutput struct { + SessionFilePath string + SourceConnectionConfig string +} + +type PrepareGcsBucket struct { + Input *PrepareGcsBucketInput + SA storageaccessor.StorageAccessor +} + +// This creates a GCS bucket if based on flag input. It subsequently uploads local files to the bucket. +func (p *PrepareGcsBucket) Transaction(ctx context.Context) error { + input := p.Input + if input.IsSMTBucketRequired { + err := resource.CreateBucketSMTResource(ctx, p.SA, input.SmtJobId, input.SmtBucketName, input.SpannerProjectId, input.SpannerLocation, nil, 45) + if err != nil { + return err + } + logger.Log.Info(fmt.Sprintf("Created bucket: %s", input.SmtBucketName)) + if !strings.HasPrefix(input.SessionFilePath, constants.GCS_FILE_PREFIX) { + err := p.SA.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "session.json", input.SessionFilePath) + if err != nil { + return fmt.Errorf("could not upload session file to GCS: %v", err) + } + logger.Log.Debug(fmt.Sprintf("Uploaded local session file: %s to bucket %s", input.SessionFilePath, input.SmtBucketName)) + } + if !strings.HasPrefix(input.SourceConnectionConfig, constants.GCS_FILE_PREFIX) { + err := p.SA.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "source-connection-config.json", input.SourceConnectionConfig) + if err != nil { + return fmt.Errorf("could not upload source connection config file to GCS: %v", err) + } + logger.Log.Debug(fmt.Sprintf("Uploaded local source connection config : %s to bucket %s", input.SourceConnectionConfig, input.SmtBucketName)) + } + } + return nil +} + +func (p *PrepareGcsBucket) Compensation(ctx context.Context) error { + return nil +} diff --git a/reverserepl/activity/prepare_metadata_db.go b/reverserepl/activity/prepare_metadata_db.go new file mode 100644 index 0000000000..f0a44b0377 --- /dev/null +++ b/reverserepl/activity/prepare_metadata_db.go @@ -0,0 +1,61 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + "fmt" + + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource" +) + +type PrepareMetadataDbInput struct { + SmtJobId string + DbURI string +} + +type PrepareMetadataDbOutput struct { + Exists bool + Created bool +} + +type PrepareMetadataDb struct { + Input *PrepareMetadataDbInput + Output *PrepareMetadataDbOutput + SpA spanneraccessor.SpannerAccessor +} + +// Creates a metadata db for reverse replication if one is not already present. +func (p *PrepareMetadataDb) Transaction(ctx context.Context) error { + input := p.Input + dbExists, err := p.SpA.CheckExistingDb(ctx, input.DbURI) + if err != nil { + return fmt.Errorf("error checking existing db: %v", err) + } + if dbExists { + logger.Log.Info(fmt.Sprintf("reverse replication metadata db %s already exists, skipping creation", input.DbURI)) + p.Output.Exists = true + return nil + } + resource.CreateMetadataDbSMTResource(ctx, p.SpA, input.SmtJobId, input.DbURI) + logger.Log.Info(fmt.Sprintf("Created reverse replication metadata db %s", input.DbURI)) + p.Output.Created = true + return nil +} + +func (p *PrepareMetadataDb) Compensation(ctx context.Context) error { + return nil +} diff --git a/reverserepl/activity/update_smt_job_entry.go b/reverserepl/activity/update_smt_job_entry.go new file mode 100644 index 0000000000..4c4b81bb72 --- /dev/null +++ b/reverserepl/activity/update_smt_job_entry.go @@ -0,0 +1,39 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package activity + +import ( + "context" + + "github.com/GoogleCloudPlatform/spanner-migration-tool/dao" +) + +type UpdateSmtJobEntryInput struct { + SmtJobId string + State string +} + +type UpdateSmtJobEntry struct { + Input *UpdateSmtJobEntryInput +} + +// Updates the state of an smt job entry. +func (p *UpdateSmtJobEntry) Transaction(ctx context.Context) error { + input := p.Input + return dao.UpdateSMTJobState(ctx, input.SmtJobId, input.State) +} + +func (p *UpdateSmtJobEntry) Compensation(ctx context.Context) error { + return nil +} diff --git a/reverserepl/create.go b/reverserepl/create.go new file mode 100644 index 0000000000..1301745b33 --- /dev/null +++ b/reverserepl/create.go @@ -0,0 +1,245 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package reverserepl + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/activity" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" + "github.com/GoogleCloudPlatform/spanner-migration-tool/dao" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + rractivity "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/activity" + "github.com/GoogleCloudPlatform/spanner-migration-tool/webv2/helpers" +) + +func validateAndUpdateJobData(ctx context.Context, request *JobData, uuid string) (err error) { + request.IsSMTBucketRequired = true + request.SmtBucketName = fmt.Sprintf("smt-rr-gcs-%s", uuid) + if strings.HasPrefix(request.SessionFilePath, constants.GCS_FILE_PREFIX) && strings.HasPrefix(request.SourceConnectionConfig, constants.GCS_FILE_PREFIX) && request.GcsDataDirectory != "" { + request.IsSMTBucketRequired = false + request.SmtBucketName = "" + } + if request.InstanceId == "" { + return fmt.Errorf("found empty InstanceId which is a required parameter") + } + if request.DatabaseId == "" { + return fmt.Errorf("found empty DatabaseId which is a required parameter") + } + if request.SessionFilePath == "" { + return fmt.Errorf("found empty SessionFilePath which is a required parameter") + } else if !strings.HasPrefix(request.SessionFilePath, constants.GCS_FILE_PREFIX) { + request.SessionFileGcsPath = fmt.Sprintf("%s%s/session.json", constants.GCS_FILE_PREFIX, request.SmtBucketName) + } else { + request.SessionFileGcsPath = request.SessionFilePath + } + if request.SourceConnectionConfig == "" { + return fmt.Errorf("found empty SourceConnectionConfig which is a required parameter") + } else if !strings.HasPrefix(request.SourceConnectionConfig, constants.GCS_FILE_PREFIX) { + request.SourceConnectionConfigGcsPath = fmt.Sprintf("%s%s/source-connection-config.json", constants.GCS_FILE_PREFIX, request.SmtBucketName) + } else { + request.SourceConnectionConfigGcsPath = request.SourceConnectionConfig + } + if request.SpannerProjectId == "" { + return fmt.Errorf("found empty SpannerProjectId which is a required parameter") + } + if request.JobName == "" { + request.JobName = fmt.Sprintf("smt-job-%s", uuid) + } + if request.SourceType == "" { + request.SourceType = constants.MYSQL + } + if request.SourceType != constants.MYSQL { + return fmt.Errorf("%s is not a valid source type for reverse replication. Only supported source type is mysql", request.SourceType) + } + if request.MetadataInstance == "" { + request.MetadataInstance = request.InstanceId + } + if request.MetadataDatabase == "" { + request.MetadataDatabase = fmt.Sprintf("smt-rr-metadata-%s", uuid) + } + if request.GcsDataDirectory == "" { + request.GcsDataDirectory = fmt.Sprintf("gs://smt-rr-gcs-%s/reverse-replication/data", uuid) + } else if !strings.HasPrefix(request.GcsDataDirectory, constants.GCS_FILE_PREFIX) { + return fmt.Errorf("invalid gcs path for GcsDataDirectory: %s", request.GcsDataDirectory) + } + if request.ChangeStreamName == "" { + request.ChangeStreamName = fmt.Sprintf("smt-rr-cs-%s", uuid) + } + if request.FiltrationMode == "" { + request.FiltrationMode = constants.RR_READER_FILTER_FWD + } else if !utils.Contains([]string{constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE}, request.FiltrationMode) { + return fmt.Errorf("found filtrationMode %s, only allowed values are [%s, %s]", request.FiltrationMode, constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE) + } + if request.TimerInterval < 1 { + request.TimerInterval = 1 + } + if request.WindowDuration == "" { + request.WindowDuration = "10s" + } + + if request.ShardingCustomJarPath != "" && request.ShardingCustomClassName == "" { + return fmt.Errorf("found non-empty value for ShardingCustomJarPath, but empty value for ShardingCustomClassName") + } + if request.ShardingCustomJarPath == "" && request.ShardingCustomClassName != "" { + return fmt.Errorf("found non-empty value for ShardingCustomClassName, but empty value for ShardingCustomJarPath") + } + if request.ShardingCustomJarPath != "" && !strings.HasPrefix(request.ShardingCustomJarPath, constants.GCS_FILE_PREFIX) { + return fmt.Errorf("please specify a valid GCS path for ShardingCustomJarPath, starting with gs://") + } + // Replace '-' with '_' since hyphens are not allowed in cs names. + request.ChangeStreamName = strings.Replace(request.ChangeStreamName, "-", "_", -1) + + request.SpannerLocation, err = spanneraccessor.GetSpannerLeaderLocation(ctx, fmt.Sprintf("projects/%s/instances/%s", request.SpannerProjectId, request.InstanceId)) + return err +} + +// CreateWorkflows sets up the data flow job and required resources for a reverse replication pipeline. +func CreateWorkflow(ctx context.Context, request JobData) error { + // Move to initialization to CLI layer. + _ = logger.InitializeLogger("DEBUG") + defer logger.Log.Sync() + + logger.Log.Info("Creating reverse replication pipeline.") + logger.Log.Debug(fmt.Sprintf("Received Create Reverse Replication job request: %+v\n", request)) + uuid := utils.GenerateHashStr() + err := validateAndUpdateJobData(ctx, &request, uuid) + if err != nil { + return fmt.Errorf("error in validateCreateRequest: %v", err) + } + logger.Log.Debug(fmt.Sprintf("Updated job request: %+v\n", request)) + + // Check or create the internal metadata database for all flows. + helpers.CheckOrCreateMetadataDb(request.SpannerProjectId, request.InstanceId) + smtMetadataDBURI := helpers.GetSpannerUri(request.SpannerProjectId, request.InstanceId) + // Init dao client. + _, err = dao.GetOrCreateClient(ctx, smtMetadataDBURI) + if err != nil { + return fmt.Errorf("error starting dao client: %v", err) + } + + smtJobId := fmt.Sprintf("smt-job-%s", uuid) + b, err := json.Marshal(request) + if err != nil { + return fmt.Errorf("error converting job data to string: %v", err) + } + jobData := string(b) + activities := []activity.Activity{ + &rractivity.CreateSmtJobEntry{ + Input: &rractivity.CreateSmtJobEntryInput{ + SmtJobId: smtJobId, + JobName: request.JobName, + SpannerProjectId: request.SpannerProjectId, + InstanceId: request.InstanceId, + DatabaseId: request.DatabaseId, + JobData: jobData, + }, + }, + &rractivity.PrepareGcsBucket{ + Input: &rractivity.PrepareGcsBucketInput{ + SmtJobId: smtJobId, + SmtBucketName: request.SmtBucketName, + SpannerProjectId: request.SpannerProjectId, + SpannerLocation: request.SpannerLocation, + SessionFilePath: request.SessionFilePath, + SourceConnectionConfig: request.SourceConnectionConfig, + IsSMTBucketRequired: request.IsSMTBucketRequired, + }, + }, + &rractivity.PrepareChangeStream{ + Input: &rractivity.PrepareChangeStreamInput{ + SmtJobId: smtJobId, + ChangeStreamName: request.ChangeStreamName, + DbURI: fmt.Sprintf("projects/%s/instances/%s/databases/%s", request.SpannerProjectId, request.InstanceId, request.DatabaseId), + }, + Output: &rractivity.PrepareChangeStreamOutput{}, + }, + &rractivity.PrepareMetadataDb{ + Input: &rractivity.PrepareMetadataDbInput{ + SmtJobId: smtJobId, + DbURI: fmt.Sprintf("projects/%s/instances/%s/databases/%s", request.SpannerProjectId, request.MetadataInstance, request.MetadataDatabase), + }, + Output: &rractivity.PrepareMetadataDbOutput{}, + }, + &rractivity.PrepareDataflowReader{ + Input: &rractivity.PrepareDataflowReaderInput{ + SmtJobId: smtJobId, + ChangeStreamName: request.ChangeStreamName, + InstanceId: request.InstanceId, + DatabaseId: request.DatabaseId, + SpannerProjectId: request.SpannerProjectId, + SessionFilePath: request.SessionFileGcsPath, + SourceShardsFilePath: request.SourceConnectionConfigGcsPath, + MetadataInstance: request.MetadataInstance, + MetadataDatabase: request.MetadataDatabase, + GcsOutputDirectory: request.GcsDataDirectory, + StartTimestamp: request.StartTimestamp, + EndTimestamp: request.EndTimestamp, + WindowDuration: request.WindowDuration, + FiltrationMode: request.FiltrationMode, + MetadataTableSuffix: request.MetadataTableSuffix, + SkipDirectoryName: request.SkipDirectoryName, + ShardingCustomJarPath: request.ShardingCustomJarPath, + ShardingCustomClassName: request.ShardingCustomClassName, + TuningCfg: request.ReaderCfg, + SpannerLocation: request.SpannerLocation, + }, + Output: &rractivity.PrepareDataflowReaderOutput{}, + }, + &rractivity.PrepareDataflowWriter{ + Input: &rractivity.PrepareDataflowWriterInput{ + SmtJobId: smtJobId, + SourceShardsFilePath: request.SourceConnectionConfigGcsPath, + SessionFilePath: request.SessionFileGcsPath, + SourceType: request.SourceType, + SourceDbTimezoneOffset: request.SourceDbTimezoneOffset, + TimerInterval: request.TimerInterval, + StartTimestamp: request.StartTimestamp, + WindowDuration: request.WindowDuration, + GCSInputDirectoryPath: request.GcsDataDirectory, + SpannerProjectId: request.SpannerProjectId, + MetadataInstance: request.MetadataInstance, + MetadataDatabase: request.MetadataDatabase, + MetadataTableSuffix: request.MetadataTableSuffix, + TuningCfg: request.WriterCfg, + SpannerLocation: request.SpannerLocation, + }, + Output: &rractivity.PrepareDataflowWriterOutput{}, + }, + &rractivity.UpdateSmtJobEntry{ + Input: &rractivity.UpdateSmtJobEntryInput{ + SmtJobId: smtJobId, + State: "RUNNING", + }, + }, + } + for i, activity := range activities { + if err := activity.Transaction(ctx); err != nil { + // If a local transaction fails, execute the compensating actions for all previous steps + // for i := len(s.Steps) - 1; i >= 0; i-- { + // if err := s.Steps[i].Compensate(); err != nil { + // return errors.New(fmt.Sprintf("failed to compensate for step %d: %v", i, err)) + // } + // } + return fmt.Errorf("error executing activity #%d: %v", i, err) + } + } + logger.Log.Info("Successfully launched reverse replication pipeline.") + return nil +} diff --git a/reverserepl/resource/resource.go b/reverserepl/resource/resource.go new file mode 100644 index 0000000000..79721fbf2b --- /dev/null +++ b/reverserepl/resource/resource.go @@ -0,0 +1,99 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package resource + +import ( + "context" + "fmt" + + "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb" + "cloud.google.com/go/spanner" + dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" + "github.com/GoogleCloudPlatform/spanner-migration-tool/dao" +) + +func CreateChangeStreamSMTResource(ctx context.Context, spA spanneraccessor.SpannerAccessor, smtJobId, changeStreamName, dbURI string) error { + resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr()) + resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_ChangeStream{DbURI: dbURI}} + err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, changeStreamName, changeStreamName, "change-stream", resourceData) + if err != nil { + return fmt.Errorf("error inserting SMT change stream resource: %v", err) + } + err = spA.CreateChangeStream(ctx, changeStreamName, dbURI) + if err != nil { + return fmt.Errorf("error in change stream creation: %v", err) + } + return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED") +} + +func CreateMetadataDbSMTResource(ctx context.Context, spA spanneraccessor.SpannerAccessor, smtJobId, dbURI string) error { + _, _, dbName := utils.ParseDbURI(dbURI) + resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr()) + resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_MetadataDb{DbURI: dbURI}} + err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, dbName, dbName, "rr-metadata-db", resourceData) + if err != nil { + return fmt.Errorf("error inserting SMT metadata db resource: %v", err) + } + err = spA.CreateEmptyDatabase(ctx, dbURI) + if err != nil { + return fmt.Errorf("error creating db: %v", err) + } + return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED") +} + +func CreateBucketSMTResource(ctx context.Context, sa storageaccessor.StorageAccessor, smtJobId, bucketName, projectId, location string, matchesPrefix []string, ttl int64) error { + resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr()) + resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_GCSBucket{ + Name: bucketName, + ProjectId: projectId, + Location: location, + MatchesPrefix: matchesPrefix, + Ttl: ttl, + }} + err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, bucketName, bucketName, "gcs-bucket", resourceData) + if err != nil { + return fmt.Errorf("error inserting SMT bucket resource: %v", err) + } + err = sa.CreateGCSBucketWithLifecycle(ctx, bucketName, projectId, location, matchesPrefix, ttl) + if err != nil { + return fmt.Errorf("error in bucket creation: %v", err) + } + return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED") +} + +func CreateDataflowSMTResource(ctx context.Context, da dataflowaccessor.DataflowAccessor, smtJobId string, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (string, error) { + resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr()) + resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_Dataflow{LaunchRequest: launchRequest, EquivalentGcloudCmd: dataflowutils.GetGcloudDataflowCommand(launchRequest)}} + err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, "", launchRequest.LaunchParameter.JobName, "dataflow", resourceData) + if err != nil { + return "", fmt.Errorf("error inserting SMT dataflow resource: %v", err) + } + response, err := da.LaunchFlexTemplate(ctx, launchRequest) + if err != nil { + return "", fmt.Errorf("error in launching dataflow job: %v", err) + } + err = dao.UpdateSMTResourceExternalId(ctx, resourceId, response.Job.Id) + if err != nil { + return "", fmt.Errorf("error updating external id for dataflow job: %v", err) + } + err = dao.UpdateSMTResourceState(ctx, resourceId, "CREATED") + if err != nil { + return "", fmt.Errorf("error updating state for dataflow job: %v", err) + } + return response.Job.Id, nil +} diff --git a/reverserepl/resource/resourcepb.go b/reverserepl/resource/resourcepb.go new file mode 100644 index 0000000000..08404f2aee --- /dev/null +++ b/reverserepl/resource/resourcepb.go @@ -0,0 +1,37 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package resource + +import "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb" + +type ResourceData_ChangeStream struct { + DbURI string `json:"dbURI"` +} + +type ResourceData_MetadataDb struct { + DbURI string `json:"dbURI"` +} + +type ResourceData_GCSBucket struct { + Name string `json:"name"` + ProjectId string `json:"projectId"` + Location string `json:"location"` + Ttl int64 `json:"ttl"` + MatchesPrefix []string `json:"matchesPrefix"` +} + +type ResourceData_Dataflow struct { + LaunchRequest *dataflowpb.LaunchFlexTemplateRequest `json:"launchRequest"` + EquivalentGcloudCmd string `json:"equivalentGcloudCmd"` +} diff --git a/reverserepl/types.go b/reverserepl/types.go new file mode 100644 index 0000000000..4884993724 --- /dev/null +++ b/reverserepl/types.go @@ -0,0 +1,51 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package reverserepl + +type JobData struct { + // Required parameters. + InstanceId string `json:"instanceId"` + DatabaseId string `json:"databaseId"` + SessionFilePath string `json:"sessionFilePath"` + SourceConnectionConfig string `json:"sourceConnectionConfig"` + SpannerProjectId string `json:"spannerProjectId"` + // Optional parameters. + JobName string `json:"jobName"` + SourceType string `json:"sourceType"` + MetadataInstance string `json:"metadataInstance"` + MetadataDatabase string `json:"metadataDatabase"` + GcsDataDirectory string `json:"gcsDataDirectory"` + ChangeStreamName string `json:"changeStreamName"` + StartTimestamp string `json:"startTimestamp"` + EndTimestamp string `json:"endTimestamp"` + WindowDuration string `json:"windowDuration"` + FiltrationMode string `json:"filtrationMode"` + SourceDbTimezoneOffset string `json:"sourceDbTimezoneOffset"` + TimerInterval int `json:"timerInterval"` + MetadataTableSuffix string `json:"metadataTableSuffix"` + SkipDirectoryName string `json:"skipDirectoryName"` + ShardingCustomJarPath string `json:"shardingCustomJarPath"` + ShardingCustomClassName string `json:"shardingCustomClassName"` + ReaderCfg string `json:"readerCfg"` + WriterCfg string `json:"writerCfg"` + // SMT generated - These fields are for use internally by SMT. These should not be configured when passing input. + IsSMTBucketRequired bool `json:"isSMTBucketRequired"` + SmtBucketName string `json:"smtBucketName"` + // Location of Spanner leader. + SpannerLocation string `json:"spannerLocation"` + // GCS location of session file path. + SessionFileGcsPath string `json:"sessionFileGcsPath"` + // GCS location of source connection config. + SourceConnectionConfigGcsPath string `json:"sourceConnectionConfigGcsPath"` +} diff --git a/streaming/streaming.go b/streaming/streaming.go index 481770b255..c25d173fdf 100644 --- a/streaming/streaming.go +++ b/streaming/streaming.go @@ -33,10 +33,13 @@ import ( resourcemanager "cloud.google.com/go/resourcemanager/apiv3" resourcemanagerpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" + dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" "github.com/GoogleCloudPlatform/spanner-migration-tool/internal" "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/GoogleCloudPlatform/spanner-migration-tool/profiles" "github.com/google/uuid" "github.com/googleapis/gax-go/v2" @@ -722,7 +725,7 @@ func LaunchDataflowJob(ctx context.Context, targetProfile profiles.TargetProfile fmt.Printf("flexTemplateRequest: %+v\n", req) return internal.DataflowOutput{}, fmt.Errorf("unable to launch template: %v", err) } - gcloudDfCmd := utils.GetGcloudDataflowCommand(req) + gcloudDfCmd := dataflowutils.GetGcloudDataflowCommand(req) logger.Log.Debug(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", req.LaunchParameter.JobName, gcloudDfCmd)) return internal.DataflowOutput{JobID: respDf.Job.Id, GCloudCmd: gcloudDfCmd}, nil } @@ -857,12 +860,12 @@ func StartDatastream(ctx context.Context, streamingCfg StreamingCfg, sourceProfi } func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, streamingCfg StreamingCfg, conv *internal.Conv) (internal.DataflowOutput, error) { - + sa := storageaccessor.StorageAccessorImpl{} convJSON, err := json.MarshalIndent(conv, "", " ") if err != nil { return internal.DataflowOutput{}, fmt.Errorf("can't encode session state to JSON: %v", err) } - err = utils.WriteToGCS(streamingCfg.TmpDir, "session.json", string(convJSON)) + err = sa.WriteDataToGCS(ctx, streamingCfg.TmpDir, "session.json", string(convJSON)) if err != nil { return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err) } @@ -873,7 +876,7 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st if err != nil { return internal.DataflowOutput{}, fmt.Errorf("failed to compute transformation context: %s", err.Error()) } - err = utils.WriteToGCS(streamingCfg.TmpDir, "transformationContext.json", string(transformationContext)) + err = sa.WriteDataToGCS(ctx, streamingCfg.TmpDir, "transformationContext.json", string(transformationContext)) if err != nil { return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err) } @@ -883,43 +886,3 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st } return dfOutput, nil } - -// Applies the bucket lifecycle with delete rule. Only accepts the Age and -// prefix rule conditions as it is only used for the Datastream destination -// bucket currently. -func EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error { - client, err := storage.NewClient(ctx) - if err != nil { - return fmt.Errorf("could not create client while enabling lifecycle: %w", err) - } - defer client.Close() - - for i, str := range matchesPrefix { - matchesPrefix[i] = strings.TrimPrefix(str, "/") - } - bucket := client.Bucket(bucketName) - bucketAttrsToUpdate := storage.BucketAttrsToUpdate{ - Lifecycle: &storage.Lifecycle{ - Rules: []storage.LifecycleRule{ - { - Action: storage.LifecycleAction{Type: "Delete"}, - Condition: storage.LifecycleCondition{ - AgeInDays: ttl, - // The prefixes should not contain the bucket names and starting slash. - // For object gs://my_bucket/pictures/paris_2022.jpg, - // you would use a condition such as "matchesPrefix":["pictures/paris_"]. - MatchesPrefix: matchesPrefix, - }, - }, - }, - }, - } - - attrs, err := bucket.Update(ctx, bucketAttrsToUpdate) - if err != nil { - return fmt.Errorf("could not bucket with lifecycle: %w", err) - } - logger.Log.Info(fmt.Sprintf("Added lifecycle rule to bucket %v\n. Rule Action: %v\t Rule Condition: %v\n", - bucketName, attrs.Lifecycle.Rules[0].Action, attrs.Lifecycle.Rules[0].Condition)) - return nil -} diff --git a/testing/accessors/spanner/spanner_accessor_test.go b/testing/accessors/spanner/spanner_accessor_test.go new file mode 100644 index 0000000000..0dd0baa07f --- /dev/null +++ b/testing/accessors/spanner/spanner_accessor_test.go @@ -0,0 +1,130 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// TODO: Refactor this file and other integration tests by moving all common code +// to remove redundancy. + +package spanneraccessor_test + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "testing" + "time" + + database "cloud.google.com/go/spanner/admin/database/apiv1" + "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" + "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" + "github.com/GoogleCloudPlatform/spanner-migration-tool/conversion" + "github.com/GoogleCloudPlatform/spanner-migration-tool/internal" + "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +var ( + projectID string + instanceID string + + ctx context.Context + databaseAdmin *database.DatabaseAdminClient +) + +// This test should move as a mock unit test inside accessors itself. +func TestMain(m *testing.M) { + cleanup := initTests() + res := m.Run() + cleanup() + os.Exit(res) +} + +func init() { + logger.Log = zap.NewNop() +} + +func initTests() (cleanup func()) { + projectID = os.Getenv("SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_PROJECT_ID") + instanceID = os.Getenv("SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_INSTANCE_ID") + + ctx = context.Background() + flag.Parse() // Needed for testing.Short(). + noop := func() {} + + if testing.Short() { + log.Println("Unit test for UpdateDDLForeignKeys skipped in -short mode.") + return noop + } + + if projectID == "" { + log.Println("Unit test for UpdateDDLForeignKeys skipped: SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_PROJECT_ID is missing") + return noop + } + + if instanceID == "" { + log.Println("Unit test for UpdateDDLForeignKeys skipped: SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_INSTANCE_ID is missing") + return noop + } + + var err error + databaseAdmin, err = database.NewDatabaseAdminClient(ctx) + if err != nil { + log.Fatalf("cannot create databaseAdmin client: %v", err) + } + + return func() { + databaseAdmin.Close() + } +} + +func dropDatabase(t *testing.T, dbPath string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + // Drop the testing database. + if err := databaseAdmin.DropDatabase(ctx, &databasepb.DropDatabaseRequest{Database: dbPath}); err != nil { + t.Fatalf("failed to drop testing database %v: %v", dbPath, err) + } +} + +func TestCheckExistingDb(t *testing.T) { + onlyRunForEmulatorTest(t) + dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, "check-db-exists") + err := conversion.CreateDatabase(ctx, databaseAdmin, dbURI, internal.MakeConv(), os.Stdout, "", constants.BULK_MIGRATION) + if err != nil { + t.Fatal(err) + } + defer dropDatabase(t, dbURI) + testCases := []struct { + dbName string + dbExists bool + }{ + {"check-db-exists", true}, + {"check-db-does-not-exist", false}, + } + spA := spanneraccessor.SpannerAccessorImpl{} + for _, tc := range testCases { + dbExists, err := spA.CheckExistingDb(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName)) + assert.Nil(t, err) + assert.Equal(t, tc.dbExists, dbExists) + } +} + +func onlyRunForEmulatorTest(t *testing.T) { + if os.Getenv("SPANNER_EMULATOR_HOST") == "" { + t.Skip("Skipping tests only running against the emulator.") + } +} diff --git a/testing/common/utils/dataflow_utils_test.go b/testing/common/utils/dataflow_utils_test.go deleted file mode 100644 index 948748c46f..0000000000 --- a/testing/common/utils/dataflow_utils_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: Refactor this file and other integration tests by moving all common code -// to remove redundancy. - -package utils_test - -import ( - "os" - "testing" - - "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb" - "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" - "github.com/stretchr/testify/assert" -) - -func TestMain(m *testing.M) { - res := m.Run() - os.Exit(res) -} - -func getTemplateDfRequest() *dataflowpb.LaunchFlexTemplateRequest { - launchParameters := &dataflowpb.LaunchFlexTemplateParameter{ - JobName: "test-job", - Template: &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://template/Cloud_Datastream_to_Spanner"}, - Parameters: map[string]string{ - "inputFilePattern": "gs://inputFilePattern", - "streamName": "my-stream", - "instanceId": "my-instance", - "databaseId": "my-dbName", - "sessionFilePath": "gs://session.json", - "deadLetterQueueDirectory": "gs://dlq", - "transformationContextFilePath": "gs://transformationContext.json", - "directoryWatchDurationInMinutes": "480", // Setting directory watch timeout to 8 hours - }, - Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{ - MaxWorkers: 50, - NumWorkers: 10, - ServiceAccountEmail: "svc-account@google.com", - TempLocation: "gs://temp-location", - MachineType: "n2-standard-16", - AdditionalExperiments: []string{"use_runner_V2", "test-experiment"}, - Network: "my-network", - Subnetwork: "my-subnetwork", - AdditionalUserLabels: map[string]string{"name": "wrench", "count": "3"}, - KmsKeyName: "sample-kms-key", - IpConfiguration: dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE, - WorkerRegion: "test-worker-region", - WorkerZone: "test-worker-zone", - EnableStreamingEngine: true, - FlexrsGoal: 1, - StagingLocation: "gs://staging-location", - }, - } - req := &dataflowpb.LaunchFlexTemplateRequest{ - ProjectId: "test-project", - LaunchParameter: launchParameters, - Location: "us-central1", - } - return req -} - -func TestGcloudCmdWithAllParams(t *testing.T) { - - req := getTemplateDfRequest() - expectedCmd := "gcloud dataflow flex-template run test-job " + - "--project=test-project --region=us-central1 " + - "--template-file-gcs-location=gs://template/Cloud_Datastream_to_Spanner " + - "--num-workers 10 --max-workers 50 --service-account-email svc-account@google.com " + - "--temp-location gs://temp-location --worker-machine-type n2-standard-16 " + - "--additional-experiments use_runner_V2,test-experiment --network my-network " + - "--subnetwork my-subnetwork --additional-user-labels {\"count\":\"3\",\"name\":\"wrench\"} " + - "--dataflow-kms-key sample-kms-key --disable-public-ips --worker-region test-worker-region " + - "--worker-zone test-worker-zone --enable-streaming-engine " + - "--flexrs-goal FLEXRS_SPEED_OPTIMIZED --staging-location gs://staging-location " + - "--parameters databaseId=my-dbName,deadLetterQueueDirectory=gs://dlq," + - "directoryWatchDurationInMinutes=480,inputFilePattern=gs://inputFilePattern," + - "instanceId=my-instance,sessionFilePath=gs://session.json,streamName=my-stream," + - "transformationContextFilePath=gs://transformationContext.json" - assert.Equal(t, expectedCmd, utils.GetGcloudDataflowCommand(req)) -} - -func TestGcloudCmdWithPartialParams(t *testing.T) { - - req := getTemplateDfRequest() - req.LaunchParameter.Parameters = make(map[string]string) - req.LaunchParameter.Environment.FlexrsGoal = 0 - req.LaunchParameter.Environment.IpConfiguration = 0 - req.LaunchParameter.Environment.EnableStreamingEngine = false - req.LaunchParameter.Environment.AdditionalExperiments = []string{} - req.LaunchParameter.Environment.AdditionalUserLabels = make(map[string]string) - req.LaunchParameter.Environment.WorkerRegion = "" - req.LaunchParameter.Environment.NumWorkers = 0 - - expectedCmd := "gcloud dataflow flex-template run test-job " + - "--project=test-project --region=us-central1 " + - "--template-file-gcs-location=gs://template/Cloud_Datastream_to_Spanner " + - "--max-workers 50 --service-account-email svc-account@google.com " + - "--temp-location gs://temp-location --worker-machine-type n2-standard-16 " + - "--network my-network --subnetwork my-subnetwork " + - "--dataflow-kms-key sample-kms-key " + - "--worker-zone test-worker-zone " + - "--staging-location gs://staging-location" - assert.Equal(t, expectedCmd, utils.GetGcloudDataflowCommand(req)) -} diff --git a/testing/conversion/conversion_test.go b/testing/conversion/conversion_test.go index ff9a8e49ef..ba08519a71 100644 --- a/testing/conversion/conversion_test.go +++ b/testing/conversion/conversion_test.go @@ -248,29 +248,6 @@ func TestVerifyDb(t *testing.T) { } } -func TestCheckExistingDb(t *testing.T) { - onlyRunForEmulatorTest(t) - dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, "check-db-exists") - err := conversion.CreateDatabase(ctx, databaseAdmin, dbURI, internal.MakeConv(), os.Stdout, "", constants.BULK_MIGRATION) - if err != nil { - t.Fatal(err) - } - defer dropDatabase(t, dbURI) - testCases := []struct { - dbName string - dbExists bool - }{ - {"check-db-exists", true}, - {"check-db-does-not-exist", false}, - } - - for _, tc := range testCases { - dbExists, err := conversion.CheckExistingDb(ctx, databaseAdmin, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName)) - assert.Nil(t, err) - assert.Equal(t, tc.dbExists, dbExists) - } -} - func TestValidateDDL(t *testing.T) { onlyRunForEmulatorTest(t) diff --git a/webv2/helpers/helpers.go b/webv2/helpers/helpers.go index d0dbea31f9..0511070cb3 100644 --- a/webv2/helpers/helpers.go +++ b/webv2/helpers/helpers.go @@ -21,8 +21,8 @@ import ( "strings" database "cloud.google.com/go/spanner/admin/database/apiv1" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" - "github.com/GoogleCloudPlatform/spanner-migration-tool/conversion" adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" ) @@ -160,7 +160,8 @@ func CheckOrCreateMetadataDb(projectId string, instanceId string) bool { } defer adminClient.Close() - dbExists, err := conversion.CheckExistingDb(ctx, adminClient, uri) + spA := spanneraccessor.SpannerAccessorImpl{} + dbExists, err := spA.CheckExistingDb(ctx, uri) if err != nil { fmt.Println(err) return false diff --git a/webv2/profile/profile.go b/webv2/profile/profile.go index 9808a594f6..923bb5d84d 100644 --- a/webv2/profile/profile.go +++ b/webv2/profile/profile.go @@ -11,6 +11,7 @@ import ( "strings" datastream "cloud.google.com/go/datastream/apiv1" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" "github.com/GoogleCloudPlatform/spanner-migration-tool/streaming" @@ -153,6 +154,7 @@ func CreateConnectionProfile(w http.ResponseWriter, r *http.Request) { ValidateOnly: details.ValidateOnly, } var bucketName string + sa := storageaccessor.StorageAccessorImpl{} if !details.IsSource { if sessionState.IsSharded { @@ -160,7 +162,7 @@ func CreateConnectionProfile(w http.ResponseWriter, r *http.Request) { } else { bucketName = strings.ToLower(sessionState.Conv.Audit.MigrationRequestId) } - err = utils.CreateGCSBucket(bucketName, sessionState.GCPProjectID, sessionState.Region) + err = sa.CreateGCSBucket(ctx, bucketName, sessionState.GCPProjectID, sessionState.Region) if err != nil { http.Error(w, fmt.Sprintf("Error while creating bucket: %v", err), http.StatusBadRequest) return diff --git a/webv2/session/session_service.go b/webv2/session/session_service.go index df190eac4c..705e2284dc 100644 --- a/webv2/session/session_service.go +++ b/webv2/session/session_service.go @@ -7,7 +7,7 @@ import ( "cloud.google.com/go/spanner" database "cloud.google.com/go/spanner/admin/database/apiv1" "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" - "github.com/GoogleCloudPlatform/spanner-migration-tool/conversion" + spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" helpers "github.com/GoogleCloudPlatform/spanner-migration-tool/webv2/helpers" ) @@ -87,8 +87,9 @@ func migrateMetadataDb(projectId, instanceId string) { } defer adminClient.Close() + spA := spanneraccessor.SpannerAccessorImpl{} oldMetadataDbUri := getOldMetadataDbUri(projectId, instanceId) - oldMetadataDBExists, err := conversion.CheckExistingDb(ctx, adminClient, oldMetadataDbUri) + oldMetadataDBExists, err := spA.CheckExistingDb(ctx, oldMetadataDbUri) if err != nil { fmt.Printf("could not check if oldMetadataDB exists. error=%v\n", err) return diff --git a/webv2/web.go b/webv2/web.go index 7b8c84b8cc..c45dab3514 100644 --- a/webv2/web.go +++ b/webv2/web.go @@ -36,6 +36,7 @@ import ( "time" instance "cloud.google.com/go/spanner/admin/instance/apiv1" + storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage" "github.com/GoogleCloudPlatform/spanner-migration-tool/cmd" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils" @@ -2263,7 +2264,7 @@ func migrate(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("Can't get source and target profiles: %v", err), http.StatusBadRequest) return } - err = writeSessionFile(sessionState) + err = writeSessionFile(ctx, sessionState) if err != nil { log.Println("can't write session file") http.Error(w, fmt.Sprintf("Can't write session file to GCS: %v", err), http.StatusBadRequest) @@ -2484,9 +2485,9 @@ func createConfigFileForShardedBulkMigration(sessionState *session.SessionState, return nil } -func writeSessionFile(sessionState *session.SessionState) error { - - err := utils.CreateGCSBucket(sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region) +func writeSessionFile(ctx context.Context, sessionState *session.SessionState) error { + sa := storageaccessor.StorageAccessorImpl{} + err := sa.CreateGCSBucket(ctx, sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region) if err != nil { return fmt.Errorf("error while creating bucket: %v", err) } @@ -2495,7 +2496,7 @@ func writeSessionFile(sessionState *session.SessionState) error { if err != nil { return fmt.Errorf("can't encode session state to JSON: %v", err) } - err = utils.WriteToGCS("gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON)) + err = sa.WriteDataToGCS(ctx, "gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON)) if err != nil { return fmt.Errorf("error while writing to GCS: %v", err) } @@ -3031,7 +3032,7 @@ type ResourceDetails struct { ResourceType string `json:"ResourceType"` ResourceName string `json:"ResourceName"` ResourceUrl string `json:"ResourceUrl"` - GcloudCmd string `json:"GcloudCmd"` + GcloudCmd string `json:"GcloudCmd"` } type GeneratedResources struct { MigrationJobId string `json:"MigrationJobId"` @@ -3054,7 +3055,7 @@ type GeneratedResources struct { AggMonitoringDashboardName string `json:"AggMonitoringDashboardName"` AggMonitoringDashboardUrl string `json:"AggMonitoringDashboardUrl"` //Used for sharded migration flow - ShardToShardResourcesMap map[string][]ResourceDetails `json:"ShardToShardResourcesMap"` + ShardToShardResourcesMap map[string][]ResourceDetails `json:"ShardToShardResourcesMap"` } func addTypeToList(convertedType string, spType string, issues []internal.SchemaIssue, l []typeIssue) []typeIssue {