From c51013a4e7c7fcb1176f131237148638d1734e97 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:11:33 +0530
Subject: [PATCH 01/35] Add dataflow accessor

---
 accessors/clients/dataflow/dataflow_client.go |  39 +++++++
 accessors/dataflow/dataflow_accessor.go       | 109 ++++++++++++++++++
 common/metrics/dashboard_components.go        |  69 ++++++-----
 common/metrics/queries.go                     |   9 +-
 common/utils/dataflow_utils.go                |  17 +--
 testing/common/utils/dataflow_utils_test.go   |   6 +-
 6 files changed, 197 insertions(+), 52 deletions(-)
 create mode 100644 accessors/clients/dataflow/dataflow_client.go
 create mode 100644 accessors/dataflow/dataflow_accessor.go

diff --git a/accessors/clients/dataflow/dataflow_client.go b/accessors/clients/dataflow/dataflow_client.go
new file mode 100644
index 0000000000..3e22c55487
--- /dev/null
+++ b/accessors/clients/dataflow/dataflow_client.go
@@ -0,0 +1,39 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dataflowclient
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	dataflow "cloud.google.com/go/dataflow/apiv1beta3"
+)
+
+var once sync.Once
+var dfClient *dataflow.FlexTemplatesClient
+
+func GetOrCreateClient(ctx context.Context) (*dataflow.FlexTemplatesClient, error) {
+	var err error
+	if dfClient == nil {
+		once.Do(func() {
+			dfClient, err = dataflow.NewFlexTemplatesClient(ctx)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create dataflow client: %v", err)
+		}
+		return dfClient, nil
+	}
+	return dfClient, nil
+}
diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
new file mode 100644
index 0000000000..308a96246a
--- /dev/null
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -0,0 +1,109 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dataflowacc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
+	dataflowclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/dataflow"
+	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+)
+
+type DataflowTuningConfig struct {
+	ProjectId             string            `json:"projectId"`
+	JobName               string            `json:"jobName"`
+	Location              string            `json:"location"`
+	VpcHostProjectId      string            `json:"hostProjectId"`
+	Network               string            `json:"network"`
+	Subnetwork            string            `json:"subnetwork"`
+	MaxWorkers            int32             `json:"maxWorkers"`
+	NumWorkers            int32             `json:"numWorkers"`
+	ServiceAccountEmail   string            `json:"serviceAccountEmail"`
+	MachineType           string            `json:"machineType"`
+	AdditionalUserLabels  map[string]string `json:"additionalUserLabels"`
+	KmsKeyName            string            `json:"kmsKeyName"`
+	GcsTemplatePath       string            `json:"gcsTemplatePath"`
+	AdditionalExperiments []string          `json:"additionalExperiments"`
+	EnableStreamingEngine bool
+}
+
+func GetDataflowLaunchRequest(parameters map[string]string, cfg DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) {
+	// If custom network is not selected, use public IP. Typical for internal testing flow.
+	vpcSubnetwork := ""
+	workerIpAddressConfig := dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PUBLIC
+	if cfg.Network != "" || cfg.Subnetwork != "" {
+		workerIpAddressConfig = dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE
+		// If subnetwork is not provided, assume network has auto subnet configuration.
+		if cfg.Subnetwork != "" {
+			if cfg.VpcHostProjectId == "" || cfg.Location == "" {
+				return nil, fmt.Errorf("vpc host project id and location must be specified when specifying subnetwork")
+			}
+			vpcSubnetwork = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", cfg.VpcHostProjectId, cfg.Location, cfg.Subnetwork)
+		}
+	}
+	request := &dataflowpb.LaunchFlexTemplateRequest{
+		ProjectId: cfg.ProjectId,
+		LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{
+			JobName:    cfg.JobName,
+			Template:   &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: cfg.GcsTemplatePath},
+			Parameters: parameters,
+			Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{
+				MaxWorkers:            cfg.MaxWorkers,
+				NumWorkers:            cfg.NumWorkers,
+				ServiceAccountEmail:   cfg.ServiceAccountEmail,
+				MachineType:           cfg.MachineType,
+				AdditionalUserLabels:  cfg.AdditionalUserLabels,
+				KmsKeyName:            cfg.KmsKeyName,
+				Network:               cfg.Network,
+				Subnetwork:            vpcSubnetwork,
+				IpConfiguration:       workerIpAddressConfig,
+				AdditionalExperiments: cfg.AdditionalExperiments,
+				EnableStreamingEngine: cfg.EnableStreamingEngine,
+			},
+		},
+		Location: cfg.Location,
+	}
+	logger.Log.Debug(fmt.Sprintf("Flex Template request generated: %+v", request))
+	return request, nil
+}
+
+func LaunchDataflowJob(ctx context.Context, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (*dataflowpb.LaunchFlexTemplateResponse, error) {
+	dfClient, err := dataflowclient.GetOrCreateClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	respDf, err := dfClient.LaunchFlexTemplate(ctx, launchRequest)
+	if err != nil {
+		logger.Log.Error(fmt.Sprintf("flexTemplateRequest: %+v\n", launchRequest))
+		return nil, fmt.Errorf("error launching dataflow template: %v", err)
+	}
+	return respDf, nil
+}
+
+func UnmarshalDataflowTuningConfig(ctx context.Context, filePath string) (DataflowTuningConfig, error) {
+	jsonStr, err := storageacc.ReadAnyFile(ctx, filePath)
+	if err != nil {
+		return DataflowTuningConfig{}, err
+	}
+	tuningCfg := DataflowTuningConfig{}
+	err = json.Unmarshal([]byte(jsonStr), &tuningCfg)
+	if err != nil {
+		return DataflowTuningConfig{}, err
+	}
+	return tuningCfg, nil
+}
diff --git a/common/metrics/dashboard_components.go b/common/metrics/dashboard_components.go
index dcd55b3358..b0e01a530b 100644
--- a/common/metrics/dashboard_components.go
+++ b/common/metrics/dashboard_components.go
@@ -4,16 +4,13 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//      http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-// Package utils contains common helper functions used across multiple other packages.
-// Utils should not import any Spanner migration tool packages.
 package metrics
 
 import (
@@ -42,22 +39,22 @@ var dashboardClient *dashboard.DashboardsClient
 
 // MonitoringMetricsResources contains information required to create the monitoring dashboard
 type MonitoringMetricsResources struct {
-	ProjectId                     string
-	DataflowJobId                 string
-	DatastreamId                  string
-	JobMetadataGcsBucket          string
-	PubsubSubscriptionId          string
-	SpannerInstanceId             string
-	SpannerDatabaseId             string
-	ShardToShardResourcesMap      map[string]internal.ShardResources
-	ShardId                       string
+	ProjectId                string
+	DataflowJobId            string
+	DatastreamId             string
+	JobMetadataGcsBucket     string
+	PubsubSubscriptionId     string
+	SpannerInstanceId        string
+	SpannerDatabaseId        string
+	ShardToShardResourcesMap map[string]internal.ShardResources
+	ShardId                  string
 	MigrationRequestId       string
 }
 
 type TileInfo struct {
 	Title             string
 	TimeSeriesQueries map[string]string // Map of legend template and their corresponding queries
-	TextContent		  string // string for text input
+	TextContent       string            // string for text input
 }
 
 type MosaicGroup struct {
@@ -95,10 +92,10 @@ func createShardDataflowMetrics(resourceIds MonitoringMetricsResources) []*dashb
 		TileInfo{
 			Title: "Dataflow Workers Memory Utilization",
 			TimeSeriesQueries: map[string]string{
-			"p50 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "50"),
-			"p90 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "90"),
-			"Max worker": fmt.Sprintf(dataflowMemoryUtilMaxQuery, resourceIds.DataflowJobId),
-		}}.createXYChartTile(),
+				"p50 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "50"),
+				"p90 worker": fmt.Sprintf(dataflowMemoryUtilPercentileQuery, resourceIds.DataflowJobId, "90"),
+				"Max worker": fmt.Sprintf(dataflowMemoryUtilMaxQuery, resourceIds.DataflowJobId),
+			}}.createXYChartTile(),
 		TileInfo{Title: "Dataflow Workers Max Backlog Time Seconds", TimeSeriesQueries: map[string]string{"": fmt.Sprintf(dataflowBacklogTimeQuery, resourceIds.DataflowJobId)}}.createXYChartTile(),
 	}
 	return dataflowTiles
@@ -107,7 +104,7 @@ func createShardDataflowMetrics(resourceIds MonitoringMetricsResources) []*dashb
 func createShardDatastreamMetrics(resourceIds MonitoringMetricsResources) []*dashboardpb.MosaicLayout_Tile {
 	datastreamTiles := []*dashboardpb.MosaicLayout_Tile{
 		TileInfo{
-			Title: "Datastream Total Latency",
+			Title:             "Datastream Total Latency",
 			TimeSeriesQueries: map[string]string{"p50 " + resourceIds.DatastreamId: fmt.Sprintf(datastreamTotalLatencyQuery, resourceIds.DatastreamId, "50"), "p90 " + resourceIds.DatastreamId: fmt.Sprintf(datastreamTotalLatencyQuery, resourceIds.DatastreamId, "90")}}.createXYChartTile(),
 		TileInfo{Title: "Datastream Throughput", TimeSeriesQueries: map[string]string{resourceIds.DatastreamId: fmt.Sprintf(datastreamThroughputQuery, resourceIds.DatastreamId)}}.createXYChartTile(),
 		TileInfo{Title: "Datastream Unsupported Events", TimeSeriesQueries: map[string]string{resourceIds.DatastreamId: fmt.Sprintf(datastreamUnsupportedEventsQuery, resourceIds.DatastreamId)}}.createXYChartTile(),
@@ -147,8 +144,8 @@ func createShardIndependentTopMetrics(resourceIds MonitoringMetricsResources) []
 		TileInfo{Title: "Datastream Unsupported Events", TimeSeriesQueries: map[string]string{resourceIds.DatastreamId: fmt.Sprintf(datastreamUnsupportedEventsQuery, resourceIds.DatastreamId)}}.createXYChartTile(),
 		TileInfo{Title: "Pubsub Age of Oldest Unacknowledged Message", TimeSeriesQueries: map[string]string{resourceIds.PubsubSubscriptionId: fmt.Sprintf(pubsubOldestUnackedMessageAgeQuery, resourceIds.PubsubSubscriptionId)}}.createXYChartTile(),
 	}
-	spannerMetrics:=createSpannerMetrics(resourceIds)
-	independentTopMetricsTiles=append(independentTopMetricsTiles,spannerMetrics...)
+	spannerMetrics := createSpannerMetrics(resourceIds)
+	independentTopMetricsTiles = append(independentTopMetricsTiles, spannerMetrics...)
 	return independentTopMetricsTiles
 }
 
@@ -178,12 +175,12 @@ func createAggDataflowMetrics(resourceIds MonitoringMetricsResources) []*dashboa
 				"Max shard": fmt.Sprintf(dataflowAggCpuUtilMaxQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs)),
 			}}.createXYChartTile(),
 		TileInfo{
-			Title: "Dataflow Workers Memory Utilization", 
+			Title: "Dataflow Workers Memory Utilization",
 			TimeSeriesQueries: map[string]string{
-			"p50 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "50"),
-			"p90 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "90"),
-			"Max shard": fmt.Sprintf(dataflowAggMemoryUtilMaxQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs)),
-		}}.createXYChartTile(),
+				"p50 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "50"),
+				"p90 shard": fmt.Sprintf(dataflowAggMemoryUtilPercentileQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs), "90"),
+				"Max shard": fmt.Sprintf(dataflowAggMemoryUtilMaxQuery, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs)),
+			}}.createXYChartTile(),
 		TileInfo{Title: "Dataflow Workers Max Backlog Time Seconds", TimeSeriesQueries: map[string]string{"Dataflow Backlog Time Seconds": fmt.Sprintf(dataflowAggBacklogTimeQuery, createAggFilterCondition("metric.job_id", dataflowJobs))}}.createXYChartTile(),
 		TileInfo{Title: "Dataflow Per Shard Median CPU Utilization", TimeSeriesQueries: map[string]string{"": fmt.Sprintf(dataflowAggPerShardCpuUtil, createAggFilterCondition("metadata.user_labels.dataflow_job_id", dataflowJobs))}}.createXYChartTile(),
 	}
@@ -197,7 +194,7 @@ func createAggDatastreamMetrics(resourceIds MonitoringMetricsResources) []*dashb
 	}
 	datastreamTiles := []*dashboardpb.MosaicLayout_Tile{
 		TileInfo{
-			Title: "Datastream Total Latency",
+			Title:             "Datastream Total Latency",
 			TimeSeriesQueries: map[string]string{"p50 Datastream Latency": fmt.Sprintf(datastreamAggTotalLatencyQuery, createAggFilterCondition("resource.stream_id", datastreamJobs), "50", "50"), "p90 Datastream Latency": fmt.Sprintf(datastreamAggTotalLatencyQuery, createAggFilterCondition("resource.stream_id", datastreamJobs), "90", "90")}}.createXYChartTile(),
 		TileInfo{Title: "Total Datastream Throughput", TimeSeriesQueries: map[string]string{"Datastream Total Throughput": fmt.Sprintf(datastreamAggThroughputQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(),
 		TileInfo{Title: "Total Datastream Unsupported Events", TimeSeriesQueries: map[string]string{"Datastream Total Unsupported Events": fmt.Sprintf(datastreamAggUnsupportedEventsQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(),
@@ -252,9 +249,9 @@ func createAggIndependentTopMetrics(resourceIds MonitoringMetricsResources) []*d
 		TileInfo{Title: "Total Datastream Throughput", TimeSeriesQueries: map[string]string{"Datastream Throughput": fmt.Sprintf(datastreamAggThroughputQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(),
 		TileInfo{Title: "Total Datastream Unsupported Events", TimeSeriesQueries: map[string]string{"Datastream Unsupported Events": fmt.Sprintf(datastreamAggUnsupportedEventsQuery, createAggFilterCondition("resource.stream_id", datastreamJobs))}}.createXYChartTile(),
 		TileInfo{Title: "Pubsub Age of Oldest Unacknowledged Message", TimeSeriesQueries: map[string]string{"Pubsub Age of Oldest Unacknowledged Message": fmt.Sprintf(pubsubAggOldestUnackedMessageAgeQuery, createAggFilterCondition("resource.subscription_id", pubsubSubs))}}.createXYChartTile(),
-		}
-	spannerMetrics:=createSpannerMetrics(resourceIds)
-	independentTopMetricsTiles=append(independentTopMetricsTiles,spannerMetrics...)
+	}
+	spannerMetrics := createSpannerMetrics(resourceIds)
+	independentTopMetricsTiles = append(independentTopMetricsTiles, spannerMetrics...)
 	return independentTopMetricsTiles
 }
 
@@ -263,7 +260,7 @@ func createAggIndependentBottomMetrics(resourceIds MonitoringMetricsResources) [
 	for shardId, shardResource := range resourceIds.ShardToShardResourcesMap {
 		shardUrl := fmt.Sprintf("https://console.cloud.google.com/monitoring/dashboards/builder/%v?project=%v", shardResource.MonitoringResources.DashboardName, resourceIds.ProjectId)
 		shardString := fmt.Sprintf("Shard [%s](%s)", shardId, shardUrl)
-		if(shardToDashboardMappingText == ""){
+		if shardToDashboardMappingText == "" {
 			shardToDashboardMappingText = shardString
 		} else {
 			shardToDashboardMappingText += " \\\n" + shardString
@@ -271,7 +268,7 @@ func createAggIndependentBottomMetrics(resourceIds MonitoringMetricsResources) [
 	}
 	independentBottomMetricsTiles := []*dashboardpb.MosaicLayout_Tile{
 		TileInfo{
-			Title: "Shard Dashboards",
+			Title:       "Shard Dashboards",
 			TextContent: shardToDashboardMappingText,
 		}.createTextTile(),
 	}
@@ -332,14 +329,14 @@ func (tileInfo TileInfo) createCollapsibleGroupTile(tiles []*dashboardpb.MosaicL
 	return &groupTile, heightOffset + groupTileHeight
 }
 
-func (tileInfo TileInfo) createTextTile() (*dashboardpb.MosaicLayout_Tile){
-	textTile :=  dashboardpb.MosaicLayout_Tile{
+func (tileInfo TileInfo) createTextTile() *dashboardpb.MosaicLayout_Tile {
+	textTile := dashboardpb.MosaicLayout_Tile{
 		Widget: &dashboardpb.Widget{
 			Title: tileInfo.Title,
 			Content: &dashboardpb.Widget_Text{
 				Text: &dashboardpb.Text{
 					Content: tileInfo.TextContent,
-					Format: dashboardpb.Text_MARKDOWN,
+					Format:  dashboardpb.Text_MARKDOWN,
 				},
 			},
 		},
@@ -382,7 +379,7 @@ func getCreateMonitoringDashboardRequest(
 	}
 
 	// create bottom independent metrics tiles
-	if createAggIndependentBottomMetrics!= nil{
+	if createAggIndependentBottomMetrics != nil {
 		independentBottomMetricsTiles := createAggIndependentBottomMetrics(resourceIds)
 		heightOffset += setWidgetPositions(independentBottomMetricsTiles, heightOffset)
 		mosaicLayoutTiles = append(mosaicLayoutTiles, independentBottomMetricsTiles...)
diff --git a/common/metrics/queries.go b/common/metrics/queries.go
index f5f124d44e..4c7dd6ea39 100644
--- a/common/metrics/queries.go
+++ b/common/metrics/queries.go
@@ -4,16 +4,13 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//      http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-// Package utils contains common helper functions used across multiple other packages.
-// Utils should not import any Spanner migration tool packages.
 package metrics
 
 // Defines queries for Monitoring Dashboard Metrics
@@ -90,8 +87,8 @@ const (
 		"filter && (%s) | group_by 1m, [value_estimated_backlog_processing_time_mean: " +
 		"mean(value.estimated_backlog_processing_time)] | every 1m | group_by [], [value_estimated_backlog_processing_time_mean_mean: " +
 		"mean(value_estimated_backlog_processing_time_mean)]"
-	dataflowAggPerShardCpuUtil = "fetch gce_instance | metric 'compute.googleapis.com/instance/cpu/utilization' | filter (%s) " + 
-		"| group_by 1m, [value_utilization_mean: mean(value.utilization)] | every 1m | group_by [metadata.user_labels.dataflow_job_id]," + 
+	dataflowAggPerShardCpuUtil = "fetch gce_instance | metric 'compute.googleapis.com/instance/cpu/utilization' | filter (%s) " +
+		"| group_by 1m, [value_utilization_mean: mean(value.utilization)] | every 1m | group_by [metadata.user_labels.dataflow_job_id]," +
 		" [value_utilization_mean_percentile: percentile(value_utilization_mean, 50)]"
 	datastreamAggThroughputQuery = "fetch datastream.googleapis.com/Stream | metric 'datastream.googleapis.com/stream/event_count' | " +
 		"filter (%s) | align rate(1m) | every 1m | group_by [], [value_event_count_aggregate: aggregate(value.event_count)]"
diff --git a/common/utils/dataflow_utils.go b/common/utils/dataflow_utils.go
index a5d4ac09f9..d302d60992 100644
--- a/common/utils/dataflow_utils.go
+++ b/common/utils/dataflow_utils.go
@@ -17,7 +17,6 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -80,11 +79,7 @@ func getEnvironmentFlags(environment *dataflowpb.FlexTemplateRuntimeEnvironment)
 		flag += fmt.Sprintf("--subnetwork %s ", environment.Subnetwork)
 	}
 	if environment.AdditionalUserLabels != nil && len(environment.AdditionalUserLabels) > 0 {
-		jsonByteStr, err := json.Marshal(environment.AdditionalUserLabels)
-		// If error is not nil, omit this flag and move on. We don't need error handling here.
-		if err == nil {
-			flag += fmt.Sprintf("--additional-user-labels %s ", string(jsonByteStr))
-		}
+		flag += fmt.Sprintf("--additional-user-labels %s ", formatAdditionalUserLabels(environment.AdditionalUserLabels))
 	}
 	if environment.KmsKeyName != "" {
 		flag += fmt.Sprintf("--dataflow-kms-key %s ", environment.KmsKeyName)
@@ -102,10 +97,18 @@ func getEnvironmentFlags(environment *dataflowpb.FlexTemplateRuntimeEnvironment)
 		flag += "--enable-streaming-engine "
 	}
 	if environment.FlexrsGoal != dataflowpb.FlexResourceSchedulingGoal_FLEXRS_UNSPECIFIED {
-		flag += fmt.Sprintf("--flexrs-goal  %s ", environment.FlexrsGoal)
+		flag += fmt.Sprintf("--flexrs-goal %s ", environment.FlexrsGoal)
 	}
 	if environment.StagingLocation != "" {
 		flag += fmt.Sprintf("--staging-location %s ", environment.StagingLocation)
 	}
 	return strings.Trim(flag, " ")
 }
+
+func formatAdditionalUserLabels(labels map[string]string) string {
+	res := []string{}
+	for key, value := range labels {
+		res = append(res, fmt.Sprintf("%s=%s", key, value))
+	}
+	return strings.Join(res, ",")
+}
diff --git a/testing/common/utils/dataflow_utils_test.go b/testing/common/utils/dataflow_utils_test.go
index 948748c46f..a54a2d705e 100644
--- a/testing/common/utils/dataflow_utils_test.go
+++ b/testing/common/utils/dataflow_utils_test.go
@@ -54,7 +54,7 @@ func getTemplateDfRequest() *dataflowpb.LaunchFlexTemplateRequest {
 			AdditionalExperiments: []string{"use_runner_V2", "test-experiment"},
 			Network:               "my-network",
 			Subnetwork:            "my-subnetwork",
-			AdditionalUserLabels:  map[string]string{"name": "wrench", "count": "3"},
+			AdditionalUserLabels:  map[string]string{"name": "wrench"},
 			KmsKeyName:            "sample-kms-key",
 			IpConfiguration:       dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE,
 			WorkerRegion:          "test-worker-region",
@@ -81,10 +81,10 @@ func TestGcloudCmdWithAllParams(t *testing.T) {
 		"--num-workers 10 --max-workers 50 --service-account-email svc-account@google.com " +
 		"--temp-location gs://temp-location --worker-machine-type n2-standard-16 " +
 		"--additional-experiments use_runner_V2,test-experiment --network my-network " +
-		"--subnetwork my-subnetwork --additional-user-labels {\"count\":\"3\",\"name\":\"wrench\"} " +
+		"--subnetwork my-subnetwork --additional-user-labels name=wrench " +
 		"--dataflow-kms-key sample-kms-key --disable-public-ips --worker-region test-worker-region " +
 		"--worker-zone test-worker-zone --enable-streaming-engine " +
-		"--flexrs-goal  FLEXRS_SPEED_OPTIMIZED --staging-location gs://staging-location " +
+		"--flexrs-goal FLEXRS_SPEED_OPTIMIZED --staging-location gs://staging-location " +
 		"--parameters databaseId=my-dbName,deadLetterQueueDirectory=gs://dlq," +
 		"directoryWatchDurationInMinutes=480,inputFilePattern=gs://inputFilePattern," +
 		"instanceId=my-instance,sessionFilePath=gs://session.json,streamName=my-stream," +

From c214468c9e96b882acd302c544a4923be94ee22c Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:29:41 +0530
Subject: [PATCH 02/35] Add enable streaming engine struct tag Mofe Unmarshall
 Method to acc2 due ot storage dependency

---
 accessors/dataflow/dataflow_accessor.go | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 308a96246a..e4ac65f08b 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -15,12 +15,10 @@ package dataflowacc
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/dataflow"
-	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 )
 
@@ -39,7 +37,7 @@ type DataflowTuningConfig struct {
 	KmsKeyName            string            `json:"kmsKeyName"`
 	GcsTemplatePath       string            `json:"gcsTemplatePath"`
 	AdditionalExperiments []string          `json:"additionalExperiments"`
-	EnableStreamingEngine bool
+	EnableStreamingEngine bool              `json:"enableStreamingEngine"`
 }
 
 func GetDataflowLaunchRequest(parameters map[string]string, cfg DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) {
@@ -94,16 +92,3 @@ func LaunchDataflowJob(ctx context.Context, launchRequest *dataflowpb.LaunchFlex
 	}
 	return respDf, nil
 }
-
-func UnmarshalDataflowTuningConfig(ctx context.Context, filePath string) (DataflowTuningConfig, error) {
-	jsonStr, err := storageacc.ReadAnyFile(ctx, filePath)
-	if err != nil {
-		return DataflowTuningConfig{}, err
-	}
-	tuningCfg := DataflowTuningConfig{}
-	err = json.Unmarshal([]byte(jsonStr), &tuningCfg)
-	if err != nil {
-		return DataflowTuningConfig{}, err
-	}
-	return tuningCfg, nil
-}

From 267ec8a3e8e48bb8b067e1cc8008c8333154c7e3 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 21:59:10 +0530
Subject: [PATCH 03/35] Moved dataflow utils to accessor and creates types.go

---
 accessors/dataflow/dataflow_accessor.go       | 62 +------------------
 accessors/dataflow/dataflow_types.go          | 32 ++++++++++
 .../dataflow}/dataflow_utils.go               | 50 +++++++++++++--
 streaming/streaming.go                        |  3 +-
 .../dataflow}/dataflow_utils_test.go          |  6 +-
 5 files changed, 83 insertions(+), 70 deletions(-)
 create mode 100644 accessors/dataflow/dataflow_types.go
 rename {common/utils => accessors/dataflow}/dataflow_utils.go (66%)
 rename testing/{common/utils => accessor/dataflow}/dataflow_utils_test.go (95%)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index e4ac65f08b..68232e788b 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -1,4 +1,4 @@
-// Copyright 2023 Google LLC
+// Copyright 2024 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package dataflowacc
+package dataflowaccessor
 
 import (
 	"context"
@@ -22,64 +22,6 @@ import (
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 )
 
-type DataflowTuningConfig struct {
-	ProjectId             string            `json:"projectId"`
-	JobName               string            `json:"jobName"`
-	Location              string            `json:"location"`
-	VpcHostProjectId      string            `json:"hostProjectId"`
-	Network               string            `json:"network"`
-	Subnetwork            string            `json:"subnetwork"`
-	MaxWorkers            int32             `json:"maxWorkers"`
-	NumWorkers            int32             `json:"numWorkers"`
-	ServiceAccountEmail   string            `json:"serviceAccountEmail"`
-	MachineType           string            `json:"machineType"`
-	AdditionalUserLabels  map[string]string `json:"additionalUserLabels"`
-	KmsKeyName            string            `json:"kmsKeyName"`
-	GcsTemplatePath       string            `json:"gcsTemplatePath"`
-	AdditionalExperiments []string          `json:"additionalExperiments"`
-	EnableStreamingEngine bool              `json:"enableStreamingEngine"`
-}
-
-func GetDataflowLaunchRequest(parameters map[string]string, cfg DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) {
-	// If custom network is not selected, use public IP. Typical for internal testing flow.
-	vpcSubnetwork := ""
-	workerIpAddressConfig := dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PUBLIC
-	if cfg.Network != "" || cfg.Subnetwork != "" {
-		workerIpAddressConfig = dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE
-		// If subnetwork is not provided, assume network has auto subnet configuration.
-		if cfg.Subnetwork != "" {
-			if cfg.VpcHostProjectId == "" || cfg.Location == "" {
-				return nil, fmt.Errorf("vpc host project id and location must be specified when specifying subnetwork")
-			}
-			vpcSubnetwork = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", cfg.VpcHostProjectId, cfg.Location, cfg.Subnetwork)
-		}
-	}
-	request := &dataflowpb.LaunchFlexTemplateRequest{
-		ProjectId: cfg.ProjectId,
-		LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{
-			JobName:    cfg.JobName,
-			Template:   &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: cfg.GcsTemplatePath},
-			Parameters: parameters,
-			Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{
-				MaxWorkers:            cfg.MaxWorkers,
-				NumWorkers:            cfg.NumWorkers,
-				ServiceAccountEmail:   cfg.ServiceAccountEmail,
-				MachineType:           cfg.MachineType,
-				AdditionalUserLabels:  cfg.AdditionalUserLabels,
-				KmsKeyName:            cfg.KmsKeyName,
-				Network:               cfg.Network,
-				Subnetwork:            vpcSubnetwork,
-				IpConfiguration:       workerIpAddressConfig,
-				AdditionalExperiments: cfg.AdditionalExperiments,
-				EnableStreamingEngine: cfg.EnableStreamingEngine,
-			},
-		},
-		Location: cfg.Location,
-	}
-	logger.Log.Debug(fmt.Sprintf("Flex Template request generated: %+v", request))
-	return request, nil
-}
-
 func LaunchDataflowJob(ctx context.Context, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (*dataflowpb.LaunchFlexTemplateResponse, error) {
 	dfClient, err := dataflowclient.GetOrCreateClient(ctx)
 	if err != nil {
diff --git a/accessors/dataflow/dataflow_types.go b/accessors/dataflow/dataflow_types.go
new file mode 100644
index 0000000000..d7b8f355ad
--- /dev/null
+++ b/accessors/dataflow/dataflow_types.go
@@ -0,0 +1,32 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dataflowaccessor
+
+type DataflowTuningConfig struct {
+	ProjectId             string            `json:"projectId"`
+	JobName               string            `json:"jobName"`
+	Location              string            `json:"location"`
+	VpcHostProjectId      string            `json:"hostProjectId"`
+	Network               string            `json:"network"`
+	Subnetwork            string            `json:"subnetwork"`
+	MaxWorkers            int32             `json:"maxWorkers"`
+	NumWorkers            int32             `json:"numWorkers"`
+	ServiceAccountEmail   string            `json:"serviceAccountEmail"`
+	MachineType           string            `json:"machineType"`
+	AdditionalUserLabels  map[string]string `json:"additionalUserLabels"`
+	KmsKeyName            string            `json:"kmsKeyName"`
+	GcsTemplatePath       string            `json:"gcsTemplatePath"`
+	AdditionalExperiments []string          `json:"additionalExperiments"`
+	EnableStreamingEngine bool              `json:"enableStreamingEngine"`
+}
diff --git a/common/utils/dataflow_utils.go b/accessors/dataflow/dataflow_utils.go
similarity index 66%
rename from common/utils/dataflow_utils.go
rename to accessors/dataflow/dataflow_utils.go
index d302d60992..b7265f1f91 100644
--- a/common/utils/dataflow_utils.go
+++ b/accessors/dataflow/dataflow_utils.go
@@ -1,20 +1,17 @@
-// Copyright 2023 Google LLC
+// Copyright 2024 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//      http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-// Package utils contains common helper functions used across multiple other packages.
-// Utils should not import any Spanner migration tool packages.
-package utils
+package dataflowaccessor
 
 import (
 	"fmt"
@@ -22,9 +19,50 @@ import (
 	"strings"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"golang.org/x/exp/maps"
 )
 
+func GetDataflowLaunchRequest(parameters map[string]string, cfg DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) {
+	// If custom network is not selected, use public IP. Typical for internal testing flow.
+	vpcSubnetwork := ""
+	workerIpAddressConfig := dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PUBLIC
+	if cfg.Network != "" || cfg.Subnetwork != "" {
+		workerIpAddressConfig = dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE
+		// If subnetwork is not provided, assume network has auto subnet configuration.
+		if cfg.Subnetwork != "" {
+			if cfg.VpcHostProjectId == "" || cfg.Location == "" {
+				return nil, fmt.Errorf("vpc host project id and location must be specified when specifying subnetwork")
+			}
+			vpcSubnetwork = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", cfg.VpcHostProjectId, cfg.Location, cfg.Subnetwork)
+		}
+	}
+	request := &dataflowpb.LaunchFlexTemplateRequest{
+		ProjectId: cfg.ProjectId,
+		LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{
+			JobName:    cfg.JobName,
+			Template:   &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: cfg.GcsTemplatePath},
+			Parameters: parameters,
+			Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{
+				MaxWorkers:            cfg.MaxWorkers,
+				NumWorkers:            cfg.NumWorkers,
+				ServiceAccountEmail:   cfg.ServiceAccountEmail,
+				MachineType:           cfg.MachineType,
+				AdditionalUserLabels:  cfg.AdditionalUserLabels,
+				KmsKeyName:            cfg.KmsKeyName,
+				Network:               cfg.Network,
+				Subnetwork:            vpcSubnetwork,
+				IpConfiguration:       workerIpAddressConfig,
+				AdditionalExperiments: cfg.AdditionalExperiments,
+				EnableStreamingEngine: cfg.EnableStreamingEngine,
+			},
+		},
+		Location: cfg.Location,
+	}
+	logger.Log.Debug(fmt.Sprintf("Flex Template request generated: %+v", request))
+	return request, nil
+}
+
 // Generate the equivalent gCloud CLI command to launch a dataflow job with the same parameters and environment flags
 // as the input body.
 func GetGcloudDataflowCommand(req *dataflowpb.LaunchFlexTemplateRequest) string {
diff --git a/streaming/streaming.go b/streaming/streaming.go
index 481770b255..296d9d3662 100644
--- a/streaming/streaming.go
+++ b/streaming/streaming.go
@@ -33,6 +33,7 @@ import (
 
 	resourcemanager "cloud.google.com/go/resourcemanager/apiv3"
 	resourcemanagerpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
@@ -722,7 +723,7 @@ func LaunchDataflowJob(ctx context.Context, targetProfile profiles.TargetProfile
 		fmt.Printf("flexTemplateRequest: %+v\n", req)
 		return internal.DataflowOutput{}, fmt.Errorf("unable to launch template: %v", err)
 	}
-	gcloudDfCmd := utils.GetGcloudDataflowCommand(req)
+	gcloudDfCmd := dataflowaccessor.GetGcloudDataflowCommand(req)
 	logger.Log.Debug(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", req.LaunchParameter.JobName, gcloudDfCmd))
 	return internal.DataflowOutput{JobID: respDf.Job.Id, GCloudCmd: gcloudDfCmd}, nil
 }
diff --git a/testing/common/utils/dataflow_utils_test.go b/testing/accessor/dataflow/dataflow_utils_test.go
similarity index 95%
rename from testing/common/utils/dataflow_utils_test.go
rename to testing/accessor/dataflow/dataflow_utils_test.go
index a54a2d705e..aec2ac85c0 100644
--- a/testing/common/utils/dataflow_utils_test.go
+++ b/testing/accessor/dataflow/dataflow_utils_test.go
@@ -22,7 +22,7 @@ import (
 	"testing"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
-	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -89,7 +89,7 @@ func TestGcloudCmdWithAllParams(t *testing.T) {
 		"directoryWatchDurationInMinutes=480,inputFilePattern=gs://inputFilePattern," +
 		"instanceId=my-instance,sessionFilePath=gs://session.json,streamName=my-stream," +
 		"transformationContextFilePath=gs://transformationContext.json"
-	assert.Equal(t, expectedCmd, utils.GetGcloudDataflowCommand(req))
+	assert.Equal(t, expectedCmd, dataflowaccessor.GetGcloudDataflowCommand(req))
 }
 
 func TestGcloudCmdWithPartialParams(t *testing.T) {
@@ -113,5 +113,5 @@ func TestGcloudCmdWithPartialParams(t *testing.T) {
 		"--dataflow-kms-key sample-kms-key " +
 		"--worker-zone test-worker-zone " +
 		"--staging-location gs://staging-location"
-	assert.Equal(t, expectedCmd, utils.GetGcloudDataflowCommand(req))
+	assert.Equal(t, expectedCmd, dataflowaccessor.GetGcloudDataflowCommand(req))
 }

From 515c2dbc4dd798a4695b15b88d07f2babd778915 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 22:24:09 +0530
Subject: [PATCH 04/35] Create dataflowutils package

---
 accessors/{ => utils}/dataflow/dataflow_utils.go | 5 +++--
 streaming/streaming.go                           | 4 ++--
 testing/accessor/dataflow/dataflow_utils_test.go | 6 +++---
 3 files changed, 8 insertions(+), 7 deletions(-)
 rename accessors/{ => utils}/dataflow/dataflow_utils.go (96%)

diff --git a/accessors/dataflow/dataflow_utils.go b/accessors/utils/dataflow/dataflow_utils.go
similarity index 96%
rename from accessors/dataflow/dataflow_utils.go
rename to accessors/utils/dataflow/dataflow_utils.go
index b7265f1f91..09906b919c 100644
--- a/accessors/dataflow/dataflow_utils.go
+++ b/accessors/utils/dataflow/dataflow_utils.go
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package dataflowaccessor
+package dataflowutils
 
 import (
 	"fmt"
@@ -19,11 +19,12 @@ import (
 	"strings"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"golang.org/x/exp/maps"
 )
 
-func GetDataflowLaunchRequest(parameters map[string]string, cfg DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) {
+func GetDataflowLaunchRequest(parameters map[string]string, cfg dataflowaccessor.DataflowTuningConfig) (*dataflowpb.LaunchFlexTemplateRequest, error) {
 	// If custom network is not selected, use public IP. Typical for internal testing flow.
 	vpcSubnetwork := ""
 	workerIpAddressConfig := dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PUBLIC
diff --git a/streaming/streaming.go b/streaming/streaming.go
index 296d9d3662..1cf1f21655 100644
--- a/streaming/streaming.go
+++ b/streaming/streaming.go
@@ -33,7 +33,7 @@ import (
 
 	resourcemanager "cloud.google.com/go/resourcemanager/apiv3"
 	resourcemanagerpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
-	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
@@ -723,7 +723,7 @@ func LaunchDataflowJob(ctx context.Context, targetProfile profiles.TargetProfile
 		fmt.Printf("flexTemplateRequest: %+v\n", req)
 		return internal.DataflowOutput{}, fmt.Errorf("unable to launch template: %v", err)
 	}
-	gcloudDfCmd := dataflowaccessor.GetGcloudDataflowCommand(req)
+	gcloudDfCmd := dataflowutils.GetGcloudDataflowCommand(req)
 	logger.Log.Debug(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", req.LaunchParameter.JobName, gcloudDfCmd))
 	return internal.DataflowOutput{JobID: respDf.Job.Id, GCloudCmd: gcloudDfCmd}, nil
 }
diff --git a/testing/accessor/dataflow/dataflow_utils_test.go b/testing/accessor/dataflow/dataflow_utils_test.go
index aec2ac85c0..834a3e8333 100644
--- a/testing/accessor/dataflow/dataflow_utils_test.go
+++ b/testing/accessor/dataflow/dataflow_utils_test.go
@@ -22,7 +22,7 @@ import (
 	"testing"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
-	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -89,7 +89,7 @@ func TestGcloudCmdWithAllParams(t *testing.T) {
 		"directoryWatchDurationInMinutes=480,inputFilePattern=gs://inputFilePattern," +
 		"instanceId=my-instance,sessionFilePath=gs://session.json,streamName=my-stream," +
 		"transformationContextFilePath=gs://transformationContext.json"
-	assert.Equal(t, expectedCmd, dataflowaccessor.GetGcloudDataflowCommand(req))
+	assert.Equal(t, expectedCmd, dataflowutils.GetGcloudDataflowCommand(req))
 }
 
 func TestGcloudCmdWithPartialParams(t *testing.T) {
@@ -113,5 +113,5 @@ func TestGcloudCmdWithPartialParams(t *testing.T) {
 		"--dataflow-kms-key sample-kms-key " +
 		"--worker-zone test-worker-zone " +
 		"--staging-location gs://staging-location"
-	assert.Equal(t, expectedCmd, dataflowaccessor.GetGcloudDataflowCommand(req))
+	assert.Equal(t, expectedCmd, dataflowutils.GetGcloudDataflowCommand(req))
 }

From 2965b6724dc4267cfcae348232792650ab219623 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 22:47:14 +0530
Subject: [PATCH 05/35] Renamed testing package for dataflow util

---
 .../utils}/dataflow/dataflow_utils_test.go                      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename testing/{accessor => accessors/utils}/dataflow/dataflow_utils_test.go (99%)

diff --git a/testing/accessor/dataflow/dataflow_utils_test.go b/testing/accessors/utils/dataflow/dataflow_utils_test.go
similarity index 99%
rename from testing/accessor/dataflow/dataflow_utils_test.go
rename to testing/accessors/utils/dataflow/dataflow_utils_test.go
index 834a3e8333..dd130b4fda 100644
--- a/testing/accessor/dataflow/dataflow_utils_test.go
+++ b/testing/accessors/utils/dataflow/dataflow_utils_test.go
@@ -15,7 +15,7 @@
 // TODO: Refactor this file and other integration tests by moving all common code
 // to remove redundancy.
 
-package utils_test
+package dataflowutils_test
 
 import (
 	"os"

From d7cca27d5fe8bea49acdba610d4e2160377f809c Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 10 Jan 2024 15:26:07 +0530
Subject: [PATCH 06/35] Added unit tests

---
 accessors/utils/dataflow/dataflow_utils.go    |   2 +
 .../utils/dataflow/dataflow_utils_test.go     | 152 +++++++++++++++---
 2 files changed, 133 insertions(+), 21 deletions(-)

diff --git a/accessors/utils/dataflow/dataflow_utils.go b/accessors/utils/dataflow/dataflow_utils.go
index 09906b919c..335fe8ed0d 100644
--- a/accessors/utils/dataflow/dataflow_utils.go
+++ b/accessors/utils/dataflow/dataflow_utils.go
@@ -38,6 +38,8 @@ func GetDataflowLaunchRequest(parameters map[string]string, cfg dataflowaccessor
 			vpcSubnetwork = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", cfg.VpcHostProjectId, cfg.Location, cfg.Subnetwork)
 		}
 	}
+	// Dataflow does not accept upper case letters in the name.
+	cfg.JobName = strings.ToLower(cfg.JobName)
 	request := &dataflowpb.LaunchFlexTemplateRequest{
 		ProjectId: cfg.ProjectId,
 		LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{
diff --git a/testing/accessors/utils/dataflow/dataflow_utils_test.go b/testing/accessors/utils/dataflow/dataflow_utils_test.go
index dd130b4fda..73c791b210 100644
--- a/testing/accessors/utils/dataflow/dataflow_utils_test.go
+++ b/testing/accessors/utils/dataflow/dataflow_utils_test.go
@@ -22,38 +22,127 @@ import (
 	"testing"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
 	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
 )
 
+func init() {
+	logger.Log = zap.NewNop()
+}
+
 func TestMain(m *testing.M) {
 	res := m.Run()
 	os.Exit(res)
 }
 
-func getTemplateDfRequest() *dataflowpb.LaunchFlexTemplateRequest {
-	launchParameters := &dataflowpb.LaunchFlexTemplateParameter{
-		JobName:  "test-job",
-		Template: &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://template/Cloud_Datastream_to_Spanner"},
-		Parameters: map[string]string{
-			"inputFilePattern":                "gs://inputFilePattern",
-			"streamName":                      "my-stream",
-			"instanceId":                      "my-instance",
-			"databaseId":                      "my-dbName",
-			"sessionFilePath":                 "gs://session.json",
-			"deadLetterQueueDirectory":        "gs://dlq",
-			"transformationContextFilePath":   "gs://transformationContext.json",
-			"directoryWatchDurationInMinutes": "480", // Setting directory watch timeout to 8 hours
+func getParameters() map[string]string {
+	return map[string]string{
+		"inputFilePattern":                "gs://inputFilePattern",
+		"streamName":                      "my-stream",
+		"instanceId":                      "my-instance",
+		"databaseId":                      "my-dbName",
+		"sessionFilePath":                 "gs://session.json",
+		"deadLetterQueueDirectory":        "gs://dlq",
+		"transformationContextFilePath":   "gs://transformationContext.json",
+		"directoryWatchDurationInMinutes": "480", // Setting directory watch timeout to 8 hours
+	}
+}
+
+func getTuningConfig() dataflowaccessor.DataflowTuningConfig {
+	return dataflowaccessor.DataflowTuningConfig{
+		ProjectId:             "test-project",
+		JobName:               "test-job",
+		Location:              "us-central1",
+		VpcHostProjectId:      "host-project",
+		Network:               "my-network",
+		Subnetwork:            "my-subnetwork",
+		MaxWorkers:            50,
+		NumWorkers:            10,
+		ServiceAccountEmail:   "svc-account@google.com",
+		MachineType:           "n2-standard-64",
+		AdditionalUserLabels:  map[string]string{"name": "wrench"},
+		KmsKeyName:            "sample-kms-key",
+		GcsTemplatePath:       "gs://template/Cloud_Datastream_to_Spanner",
+		AdditionalExperiments: []string{"use_runner_V2", "test-experiment"},
+		EnableStreamingEngine: true,
+	}
+}
+
+func getTemplateDfRequest1() *dataflowpb.LaunchFlexTemplateRequest {
+	return &dataflowpb.LaunchFlexTemplateRequest{
+		ProjectId: "test-project",
+		Location:  "us-central1",
+		LaunchParameter: &dataflowpb.LaunchFlexTemplateParameter{
+			JobName:    "test-job",
+			Template:   &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://template/Cloud_Datastream_to_Spanner"},
+			Parameters: getParameters(),
+			Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{
+				MaxWorkers:            50,
+				NumWorkers:            10,
+				ServiceAccountEmail:   "svc-account@google.com",
+				MachineType:           "n2-standard-64",
+				AdditionalUserLabels:  map[string]string{"name": "wrench"},
+				KmsKeyName:            "sample-kms-key",
+				Network:               "my-network",
+				Subnetwork:            "https://www.googleapis.com/compute/v1/projects/host-project/regions/us-central1/subnetworks/my-subnetwork",
+				IpConfiguration:       dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE,
+				AdditionalExperiments: []string{"use_runner_V2", "test-experiment"},
+				EnableStreamingEngine: true,
+			},
 		},
+	}
+}
+
+func TestGetDataflowLaunchRequestBasic(t *testing.T) {
+	params := getParameters()
+	cfg := getTuningConfig()
+	actual, err := dataflowutils.GetDataflowLaunchRequest(params, cfg)
+	if err != nil {
+		t.Fail()
+	}
+	expected := getTemplateDfRequest1()
+	assert.True(t, EquateLaunchFlexTemplateRequest(expected, actual))
+}
+
+func TestGetDataflowLaunchRequestMissingVpcHost(t *testing.T) {
+	params := getParameters()
+	cfg := getTuningConfig()
+	cfg.VpcHostProjectId = ""
+	_, err := dataflowutils.GetDataflowLaunchRequest(params, cfg)
+	assert.True(t, err != nil)
+}
+
+func TestGetDataflowLaunchRequestNameToLowerCase(t *testing.T) {
+	params := getParameters()
+	cfg := getTuningConfig()
+	cfg.JobName = "CAPITalJobName"
+	actual, err := dataflowutils.GetDataflowLaunchRequest(params, cfg)
+	if err != nil {
+		t.Fail()
+	}
+	expected := getTemplateDfRequest1()
+	expected.LaunchParameter.JobName = "capitaljobname"
+	assert.True(t, EquateLaunchFlexTemplateRequest(expected, actual))
+}
+
+func getTemplateDfRequest2() *dataflowpb.LaunchFlexTemplateRequest {
+	launchParameters := &dataflowpb.LaunchFlexTemplateParameter{
+		JobName:    "test-job",
+		Template:   &dataflowpb.LaunchFlexTemplateParameter_ContainerSpecGcsPath{ContainerSpecGcsPath: "gs://template/Cloud_Datastream_to_Spanner"},
+		Parameters: getParameters(),
 		Environment: &dataflowpb.FlexTemplateRuntimeEnvironment{
 			MaxWorkers:            50,
 			NumWorkers:            10,
 			ServiceAccountEmail:   "svc-account@google.com",
 			TempLocation:          "gs://temp-location",
-			MachineType:           "n2-standard-16",
+			MachineType:           "n2-standard-64",
 			AdditionalExperiments: []string{"use_runner_V2", "test-experiment"},
 			Network:               "my-network",
-			Subnetwork:            "my-subnetwork",
+			Subnetwork:            "https://www.googleapis.com/compute/v1/projects/host-project/regions/us-central1/subnetworks/my-subnetwork",
 			AdditionalUserLabels:  map[string]string{"name": "wrench"},
 			KmsKeyName:            "sample-kms-key",
 			IpConfiguration:       dataflowpb.WorkerIPAddressConfiguration_WORKER_IP_PRIVATE,
@@ -74,14 +163,14 @@ func getTemplateDfRequest() *dataflowpb.LaunchFlexTemplateRequest {
 
 func TestGcloudCmdWithAllParams(t *testing.T) {
 
-	req := getTemplateDfRequest()
+	req := getTemplateDfRequest2()
 	expectedCmd := "gcloud dataflow flex-template run test-job " +
 		"--project=test-project --region=us-central1 " +
 		"--template-file-gcs-location=gs://template/Cloud_Datastream_to_Spanner " +
 		"--num-workers 10 --max-workers 50 --service-account-email svc-account@google.com " +
-		"--temp-location gs://temp-location --worker-machine-type n2-standard-16 " +
+		"--temp-location gs://temp-location --worker-machine-type n2-standard-64 " +
 		"--additional-experiments use_runner_V2,test-experiment --network my-network " +
-		"--subnetwork my-subnetwork --additional-user-labels name=wrench " +
+		"--subnetwork https://www.googleapis.com/compute/v1/projects/host-project/regions/us-central1/subnetworks/my-subnetwork --additional-user-labels name=wrench " +
 		"--dataflow-kms-key sample-kms-key --disable-public-ips --worker-region test-worker-region " +
 		"--worker-zone test-worker-zone --enable-streaming-engine " +
 		"--flexrs-goal FLEXRS_SPEED_OPTIMIZED --staging-location gs://staging-location " +
@@ -94,7 +183,7 @@ func TestGcloudCmdWithAllParams(t *testing.T) {
 
 func TestGcloudCmdWithPartialParams(t *testing.T) {
 
-	req := getTemplateDfRequest()
+	req := getTemplateDfRequest2()
 	req.LaunchParameter.Parameters = make(map[string]string)
 	req.LaunchParameter.Environment.FlexrsGoal = 0
 	req.LaunchParameter.Environment.IpConfiguration = 0
@@ -103,15 +192,36 @@ func TestGcloudCmdWithPartialParams(t *testing.T) {
 	req.LaunchParameter.Environment.AdditionalUserLabels = make(map[string]string)
 	req.LaunchParameter.Environment.WorkerRegion = ""
 	req.LaunchParameter.Environment.NumWorkers = 0
+	req.LaunchParameter.Environment.Network = ""
+	req.LaunchParameter.Environment.Subnetwork = ""
 
 	expectedCmd := "gcloud dataflow flex-template run test-job " +
 		"--project=test-project --region=us-central1 " +
 		"--template-file-gcs-location=gs://template/Cloud_Datastream_to_Spanner " +
 		"--max-workers 50 --service-account-email svc-account@google.com " +
-		"--temp-location gs://temp-location --worker-machine-type n2-standard-16 " +
-		"--network my-network --subnetwork my-subnetwork " +
+		"--temp-location gs://temp-location --worker-machine-type n2-standard-64 " +
 		"--dataflow-kms-key sample-kms-key " +
 		"--worker-zone test-worker-zone " +
 		"--staging-location gs://staging-location"
 	assert.Equal(t, expectedCmd, dataflowutils.GetGcloudDataflowCommand(req))
 }
+
+func EquateLaunchFlexTemplateRequest(df1 *dataflowpb.LaunchFlexTemplateRequest, df2 *dataflowpb.LaunchFlexTemplateRequest) bool {
+	lp1 := df1.LaunchParameter
+	lp2 := df2.LaunchParameter
+	return (df1.ProjectId == df2.ProjectId &&
+		df1.Location == df2.Location &&
+		lp1.JobName == lp2.JobName &&
+		lp1.Environment.MaxWorkers == lp2.Environment.MaxWorkers &&
+		lp1.Environment.NumWorkers == lp2.Environment.NumWorkers &&
+		lp1.Environment.ServiceAccountEmail == lp2.Environment.ServiceAccountEmail &&
+		lp1.Environment.MachineType == lp2.Environment.MachineType &&
+		lp1.Environment.KmsKeyName == lp2.Environment.KmsKeyName &&
+		lp1.Environment.Network == lp2.Environment.Network &&
+		lp1.Environment.Subnetwork == lp2.Environment.Subnetwork &&
+		lp1.Environment.GetIpConfiguration().String() == lp2.Environment.GetIpConfiguration().String() &&
+		lp1.Environment.EnableStreamingEngine == lp2.Environment.EnableStreamingEngine &&
+		cmp.Equal(lp1.Environment.AdditionalUserLabels, lp2.Environment.AdditionalUserLabels) &&
+		cmp.Equal(lp1.Environment.AdditionalExperiments, lp2.Environment.AdditionalExperiments) &&
+		lp1.GetContainerSpecGcsPath() == lp2.GetContainerSpecGcsPath())
+}

From 60fe81108effbcef7c909aed26139019f531b9df Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 10 Jan 2024 19:48:38 +0530
Subject: [PATCH 07/35] Added empty test files for clients

---
 .../clients/dataflow/dataflow_client_test.go       | 14 ++++++++++++++
 .../accessors/dataflow/dataflow_accessor_test.go   | 14 ++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 testing/accessors/clients/dataflow/dataflow_client_test.go
 create mode 100644 testing/accessors/dataflow/dataflow_accessor_test.go

diff --git a/testing/accessors/clients/dataflow/dataflow_client_test.go b/testing/accessors/clients/dataflow/dataflow_client_test.go
new file mode 100644
index 0000000000..2a80cbf27d
--- /dev/null
+++ b/testing/accessors/clients/dataflow/dataflow_client_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dataflowclient_test
diff --git a/testing/accessors/dataflow/dataflow_accessor_test.go b/testing/accessors/dataflow/dataflow_accessor_test.go
new file mode 100644
index 0000000000..c1979843d5
--- /dev/null
+++ b/testing/accessors/dataflow/dataflow_accessor_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dataflowaccessor_test

From fab0bc41cbc2f01a844a943a047d748a28dc442e Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 10 Jan 2024 20:15:01 +0530
Subject: [PATCH 08/35] Move test to same package

---
 .../clients/dataflow/dataflow_client_test.go                      | 0
 .../accessors => accessors}/dataflow/dataflow_accessor_test.go    | 0
 .../accessors => accessors}/utils/dataflow/dataflow_utils_test.go | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename {testing/accessors => accessors}/clients/dataflow/dataflow_client_test.go (100%)
 rename {testing/accessors => accessors}/dataflow/dataflow_accessor_test.go (100%)
 rename {testing/accessors => accessors}/utils/dataflow/dataflow_utils_test.go (100%)

diff --git a/testing/accessors/clients/dataflow/dataflow_client_test.go b/accessors/clients/dataflow/dataflow_client_test.go
similarity index 100%
rename from testing/accessors/clients/dataflow/dataflow_client_test.go
rename to accessors/clients/dataflow/dataflow_client_test.go
diff --git a/testing/accessors/dataflow/dataflow_accessor_test.go b/accessors/dataflow/dataflow_accessor_test.go
similarity index 100%
rename from testing/accessors/dataflow/dataflow_accessor_test.go
rename to accessors/dataflow/dataflow_accessor_test.go
diff --git a/testing/accessors/utils/dataflow/dataflow_utils_test.go b/accessors/utils/dataflow/dataflow_utils_test.go
similarity index 100%
rename from testing/accessors/utils/dataflow/dataflow_utils_test.go
rename to accessors/utils/dataflow/dataflow_utils_test.go

From 820fc08599da334a215e9739587c2a682318619a Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 23 Jan 2024 11:34:37 +0530
Subject: [PATCH 09/35] Add tests for dataflow client

---
 accessors/clients/dataflow/dataflow_client.go |  3 +-
 .../clients/dataflow/dataflow_client_test.go  | 88 ++++++++++++++++++-
 accessors/dataflow/dataflow_accessor_test.go  |  2 +-
 .../utils/dataflow/dataflow_utils_test.go     | 19 ++--
 4 files changed, 97 insertions(+), 15 deletions(-)

diff --git a/accessors/clients/dataflow/dataflow_client.go b/accessors/clients/dataflow/dataflow_client.go
index 3e22c55487..81b821f42f 100644
--- a/accessors/clients/dataflow/dataflow_client.go
+++ b/accessors/clients/dataflow/dataflow_client.go
@@ -23,12 +23,13 @@ import (
 
 var once sync.Once
 var dfClient *dataflow.FlexTemplatesClient
+var newFlexTemplatesClient = dataflow.NewFlexTemplatesClient
 
 func GetOrCreateClient(ctx context.Context) (*dataflow.FlexTemplatesClient, error) {
 	var err error
 	if dfClient == nil {
 		once.Do(func() {
-			dfClient, err = dataflow.NewFlexTemplatesClient(ctx)
+			dfClient, err = newFlexTemplatesClient(ctx)
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create dataflow client: %v", err)
diff --git a/accessors/clients/dataflow/dataflow_client_test.go b/accessors/clients/dataflow/dataflow_client_test.go
index 2a80cbf27d..da481e8e13 100644
--- a/accessors/clients/dataflow/dataflow_client_test.go
+++ b/accessors/clients/dataflow/dataflow_client_test.go
@@ -11,4 +11,90 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package dataflowclient_test
+package dataflowclient
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+
+	dataflow "cloud.google.com/go/dataflow/apiv1beta3"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func resetTest() {
+	dfClient = nil
+	once = sync.Once{}
+}
+
+func TestGetOrCreateClient_Basic(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+	dfClient = nil
+
+	oldFunc := newFlexTemplatesClient
+	defer func() { newFlexTemplatesClient = oldFunc }()
+	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err = GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldC, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, oldC)
+	assert.Nil(t, err)
+
+	once = sync.Once{}
+	oldFunc := newFlexTemplatesClient
+	defer func() { newFlexTemplatesClient = oldFunc }()
+	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	newC, err := GetOrCreateClient(ctx)
+	assert.Equal(t, oldC, newC)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_Error(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newFlexTemplatesClient
+	defer func() { newFlexTemplatesClient = oldFunc }()
+
+	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.NotNil(t, err)
+}
diff --git a/accessors/dataflow/dataflow_accessor_test.go b/accessors/dataflow/dataflow_accessor_test.go
index c1979843d5..9b9ed5a4e7 100644
--- a/accessors/dataflow/dataflow_accessor_test.go
+++ b/accessors/dataflow/dataflow_accessor_test.go
@@ -11,4 +11,4 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package dataflowaccessor_test
+package dataflowaccessor
diff --git a/accessors/utils/dataflow/dataflow_utils_test.go b/accessors/utils/dataflow/dataflow_utils_test.go
index 73c791b210..a6bc771c9c 100644
--- a/accessors/utils/dataflow/dataflow_utils_test.go
+++ b/accessors/utils/dataflow/dataflow_utils_test.go
@@ -4,18 +4,14 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//      http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-// TODO: Refactor this file and other integration tests by moving all common code
-// to remove redundancy.
-
-package dataflowutils_test
+package dataflowutils
 
 import (
 	"os"
@@ -23,7 +19,6 @@ import (
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
-	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
@@ -100,7 +95,7 @@ func getTemplateDfRequest1() *dataflowpb.LaunchFlexTemplateRequest {
 func TestGetDataflowLaunchRequestBasic(t *testing.T) {
 	params := getParameters()
 	cfg := getTuningConfig()
-	actual, err := dataflowutils.GetDataflowLaunchRequest(params, cfg)
+	actual, err := GetDataflowLaunchRequest(params, cfg)
 	if err != nil {
 		t.Fail()
 	}
@@ -112,7 +107,7 @@ func TestGetDataflowLaunchRequestMissingVpcHost(t *testing.T) {
 	params := getParameters()
 	cfg := getTuningConfig()
 	cfg.VpcHostProjectId = ""
-	_, err := dataflowutils.GetDataflowLaunchRequest(params, cfg)
+	_, err := GetDataflowLaunchRequest(params, cfg)
 	assert.True(t, err != nil)
 }
 
@@ -120,7 +115,7 @@ func TestGetDataflowLaunchRequestNameToLowerCase(t *testing.T) {
 	params := getParameters()
 	cfg := getTuningConfig()
 	cfg.JobName = "CAPITalJobName"
-	actual, err := dataflowutils.GetDataflowLaunchRequest(params, cfg)
+	actual, err := GetDataflowLaunchRequest(params, cfg)
 	if err != nil {
 		t.Fail()
 	}
@@ -178,7 +173,7 @@ func TestGcloudCmdWithAllParams(t *testing.T) {
 		"directoryWatchDurationInMinutes=480,inputFilePattern=gs://inputFilePattern," +
 		"instanceId=my-instance,sessionFilePath=gs://session.json,streamName=my-stream," +
 		"transformationContextFilePath=gs://transformationContext.json"
-	assert.Equal(t, expectedCmd, dataflowutils.GetGcloudDataflowCommand(req))
+	assert.Equal(t, expectedCmd, GetGcloudDataflowCommand(req))
 }
 
 func TestGcloudCmdWithPartialParams(t *testing.T) {
@@ -203,7 +198,7 @@ func TestGcloudCmdWithPartialParams(t *testing.T) {
 		"--dataflow-kms-key sample-kms-key " +
 		"--worker-zone test-worker-zone " +
 		"--staging-location gs://staging-location"
-	assert.Equal(t, expectedCmd, dataflowutils.GetGcloudDataflowCommand(req))
+	assert.Equal(t, expectedCmd, GetGcloudDataflowCommand(req))
 }
 
 func EquateLaunchFlexTemplateRequest(df1 *dataflowpb.LaunchFlexTemplateRequest, df2 *dataflowpb.LaunchFlexTemplateRequest) bool {

From 84c1f7418173ad0546b213d92bf5292e5ae1d26b Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 23 Jan 2024 15:38:47 +0530
Subject: [PATCH 10/35] Update fake for client test

---
 .../clients/dataflow/dataflow_client_test.go  | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/accessors/clients/dataflow/dataflow_client_test.go b/accessors/clients/dataflow/dataflow_client_test.go
index da481e8e13..c4fbf34da1 100644
--- a/accessors/clients/dataflow/dataflow_client_test.go
+++ b/accessors/clients/dataflow/dataflow_client_test.go
@@ -44,6 +44,11 @@ func resetTest() {
 func TestGetOrCreateClient_Basic(t *testing.T) {
 	resetTest()
 	ctx := context.Background()
+	oldFunc := newFlexTemplatesClient
+	defer func() { newFlexTemplatesClient = oldFunc }()
+	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
+		return &dataflow.FlexTemplatesClient{}, nil
+	}
 	c, err := GetOrCreateClient(ctx)
 	assert.NotNil(t, c)
 	assert.Nil(t, err)
@@ -52,13 +57,17 @@ func TestGetOrCreateClient_Basic(t *testing.T) {
 func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
 	resetTest()
 	ctx := context.Background()
+	oldFunc := newFlexTemplatesClient
+	defer func() { newFlexTemplatesClient = oldFunc }()
+
+	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
+		return &dataflow.FlexTemplatesClient{}, nil
+	}
 	c, err := GetOrCreateClient(ctx)
 	assert.NotNil(t, c)
 	assert.Nil(t, err)
 	dfClient = nil
 
-	oldFunc := newFlexTemplatesClient
-	defer func() { newFlexTemplatesClient = oldFunc }()
 	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
 		return nil, fmt.Errorf("test error")
 	}
@@ -70,13 +79,17 @@ func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
 func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
 	resetTest()
 	ctx := context.Background()
+	oldFunc := newFlexTemplatesClient
+	defer func() { newFlexTemplatesClient = oldFunc }()
+
+	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
+		return &dataflow.FlexTemplatesClient{}, nil
+	}
 	oldC, err := GetOrCreateClient(ctx)
 	assert.NotNil(t, oldC)
 	assert.Nil(t, err)
 
 	once = sync.Once{}
-	oldFunc := newFlexTemplatesClient
-	defer func() { newFlexTemplatesClient = oldFunc }()
 	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
 		return nil, fmt.Errorf("test error")
 	}

From 86daf581075742e41cd750e36ec54e057f00be44 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 23 Jan 2024 16:15:50 +0530
Subject: [PATCH 11/35] Make dataflow accessor interface and struct to make it
 testable

---
 accessors/dataflow/dataflow_accessor.go | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 68232e788b..18e0f4124b 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -20,16 +20,27 @@ import (
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/googleapis/gax-go/v2"
 )
 
-func LaunchDataflowJob(ctx context.Context, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (*dataflowpb.LaunchFlexTemplateResponse, error) {
+type DataflowAccessor interface {
+	LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error)
+}
+
+func NewDataflowAccessor() DataflowAccessor {
+	return DataflowAccessorImpl{}
+}
+
+type DataflowAccessorImpl struct{}
+
+func (dfA DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) {
 	dfClient, err := dataflowclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return nil, err
 	}
-	respDf, err := dfClient.LaunchFlexTemplate(ctx, launchRequest)
+	respDf, err := dfClient.LaunchFlexTemplate(ctx, req)
 	if err != nil {
-		logger.Log.Error(fmt.Sprintf("flexTemplateRequest: %+v\n", launchRequest))
+		logger.Log.Error(fmt.Sprintf("flexTemplateRequest: %+v\n", req))
 		return nil, fmt.Errorf("error launching dataflow template: %v", err)
 	}
 	return respDf, nil

From 1f17260abd659a4855b2ea853b1ccd1ba611c70e Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 23 Jan 2024 17:06:23 +0530
Subject: [PATCH 12/35] Remove interface from accessor package

---
 accessors/dataflow/dataflow_accessor.go | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 18e0f4124b..8b582c9eec 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -23,14 +23,6 @@ import (
 	"github.com/googleapis/gax-go/v2"
 )
 
-type DataflowAccessor interface {
-	LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error)
-}
-
-func NewDataflowAccessor() DataflowAccessor {
-	return DataflowAccessorImpl{}
-}
-
 type DataflowAccessorImpl struct{}
 
 func (dfA DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) {

From fb9cd9a18b9997842d6721d1f006a1ff53f7f5a9 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 13:44:17 +0530
Subject: [PATCH 13/35] Add dataflow accessor interface

---
 accessors/dataflow/dataflow_accessor.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 8b582c9eec..28d10c68da 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -23,6 +23,10 @@ import (
 	"github.com/googleapis/gax-go/v2"
 )
 
+type DataflowAccessor interface {
+	LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error)
+}
+
 type DataflowAccessorImpl struct{}
 
 func (dfA DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) {

From ae27ee4f11834ee7ee12ca041349483a72617427 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 14:05:15 +0530
Subject: [PATCH 14/35] Add comments to dataflow client and comments on unit
 tests

---
 accessors/clients/dataflow/dataflow_client.go      | 3 +++
 accessors/clients/dataflow/dataflow_client_test.go | 5 ++++-
 accessors/dataflow/dataflow_accessor.go            | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/accessors/clients/dataflow/dataflow_client.go b/accessors/clients/dataflow/dataflow_client.go
index 81b821f42f..94ac4584f7 100644
--- a/accessors/clients/dataflow/dataflow_client.go
+++ b/accessors/clients/dataflow/dataflow_client.go
@@ -23,6 +23,9 @@ import (
 
 var once sync.Once
 var dfClient *dataflow.FlexTemplatesClient
+
+// This function is declared as a global variable to make it testable. The unit
+// tests edit this function, acting like a double.
 var newFlexTemplatesClient = dataflow.NewFlexTemplatesClient
 
 func GetOrCreateClient(ctx context.Context) (*dataflow.FlexTemplatesClient, error) {
diff --git a/accessors/clients/dataflow/dataflow_client_test.go b/accessors/clients/dataflow/dataflow_client_test.go
index c4fbf34da1..a76cc98fc4 100644
--- a/accessors/clients/dataflow/dataflow_client_test.go
+++ b/accessors/clients/dataflow/dataflow_client_test.go
@@ -66,8 +66,9 @@ func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
 	c, err := GetOrCreateClient(ctx)
 	assert.NotNil(t, c)
 	assert.Nil(t, err)
+	// Explicitly set the client to nil. Running GetOrCreateClient should not create a
+	// new client since sync would already be executed.
 	dfClient = nil
-
 	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
 		return nil, fmt.Errorf("test error")
 	}
@@ -89,6 +90,8 @@ func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
 	assert.NotNil(t, oldC)
 	assert.Nil(t, err)
 
+	// Explicitly reset once. Running GetOrCreateClient should not create a
+	// new client the if condition should prevent it.
 	once = sync.Once{}
 	newFlexTemplatesClient = func(ctx context.Context, opts ...option.ClientOption) (*dataflow.FlexTemplatesClient, error) {
 		return nil, fmt.Errorf("test error")
diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 28d10c68da..3cffedaaba 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -29,7 +29,7 @@ type DataflowAccessor interface {
 
 type DataflowAccessorImpl struct{}
 
-func (dfA DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) {
+func (dfA *DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *dataflowpb.LaunchFlexTemplateRequest, opts ...gax.CallOption) (*dataflowpb.LaunchFlexTemplateResponse, error) {
 	dfClient, err := dataflowclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return nil, err

From f71b602c8e799276c878bfd6dfad159543b3fe90 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:16:18 +0530
Subject: [PATCH 15/35] Add accessors for storage and spanner.

---
 .../clients/spanner/admin/admin_client.go     |  39 ++++
 .../clients/spanner/client/spanner_client.go  |  39 ++++
 .../instanceadmin/spanner_instance_admin.go   |  39 ++++
 accessors/clients/storage/storage_client.go   |  39 ++++
 accessors/spanner/spanner_accessor.go         | 203 ++++++++++++++++++
 accessors/storage/storage_accessor.go         | 192 +++++++++++++++++
 cmd/data.go                                   |   3 +-
 common/utils/storage_utils.go                 |  41 ++++
 common/utils/utils.go                         |  87 +-------
 conversion/conversion.go                      |  43 +---
 streaming/streaming.go                        |  46 +---
 .../spanner/spanner_accessor_test.go          | 129 +++++++++++
 testing/conversion/conversion_test.go         |  23 --
 webv2/helpers/helpers.go                      |   4 +-
 webv2/profile/profile.go                      |   3 +-
 webv2/session/session_service.go              |   4 +-
 webv2/web.go                                  |  13 +-
 17 files changed, 758 insertions(+), 189 deletions(-)
 create mode 100644 accessors/clients/spanner/admin/admin_client.go
 create mode 100644 accessors/clients/spanner/client/spanner_client.go
 create mode 100644 accessors/clients/spanner/instanceadmin/spanner_instance_admin.go
 create mode 100644 accessors/clients/storage/storage_client.go
 create mode 100644 accessors/spanner/spanner_accessor.go
 create mode 100644 accessors/storage/storage_accessor.go
 create mode 100644 common/utils/storage_utils.go
 create mode 100644 testing/accessors/spanner/spanner_accessor_test.go

diff --git a/accessors/clients/spanner/admin/admin_client.go b/accessors/clients/spanner/admin/admin_client.go
new file mode 100644
index 0000000000..c0ba8aed5f
--- /dev/null
+++ b/accessors/clients/spanner/admin/admin_client.go
@@ -0,0 +1,39 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spanneradmin
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	database "cloud.google.com/go/spanner/admin/database/apiv1"
+)
+
+var once sync.Once
+var spannerAdminClient *database.DatabaseAdminClient
+
+func GetOrCreateClient(ctx context.Context) (*database.DatabaseAdminClient, error) {
+	var err error
+	if spannerAdminClient == nil {
+		once.Do(func() {
+			spannerAdminClient, err = database.NewDatabaseAdminClient(ctx)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create spanner admin client: %v", err)
+		}
+		return spannerAdminClient, nil
+	}
+	return spannerAdminClient, nil
+}
diff --git a/accessors/clients/spanner/client/spanner_client.go b/accessors/clients/spanner/client/spanner_client.go
new file mode 100644
index 0000000000..bbb3e252c1
--- /dev/null
+++ b/accessors/clients/spanner/client/spanner_client.go
@@ -0,0 +1,39 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spannerclient
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	sp "cloud.google.com/go/spanner"
+)
+
+var once sync.Once
+var spannerClient *sp.Client
+
+func GetOrCreateClient(ctx context.Context, dbURI string) (*sp.Client, error) {
+	var err error
+	if spannerClient == nil || spannerClient.DatabaseName() != dbURI {
+		once.Do(func() {
+			spannerClient, err = sp.NewClient(ctx, dbURI)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create spanner database client: %v", err)
+		}
+		return spannerClient, nil
+	}
+	return spannerClient, nil
+}
diff --git a/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go b/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go
new file mode 100644
index 0000000000..324da4ac32
--- /dev/null
+++ b/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go
@@ -0,0 +1,39 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spinstanceadmin
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	instance "cloud.google.com/go/spanner/admin/instance/apiv1"
+)
+
+var once sync.Once
+var instanceAdminClient *instance.InstanceAdminClient
+
+func GetOrCreateClient(ctx context.Context) (*instance.InstanceAdminClient, error) {
+	var err error
+	if instanceAdminClient == nil {
+		once.Do(func() {
+			instanceAdminClient, err = instance.NewInstanceAdminClient(ctx)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create spanner instance admin client: %v", err)
+		}
+		return instanceAdminClient, nil
+	}
+	return instanceAdminClient, nil
+}
diff --git a/accessors/clients/storage/storage_client.go b/accessors/clients/storage/storage_client.go
new file mode 100644
index 0000000000..569841d299
--- /dev/null
+++ b/accessors/clients/storage/storage_client.go
@@ -0,0 +1,39 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package storageclient
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"cloud.google.com/go/storage"
+)
+
+var once sync.Once
+var gcsClient *storage.Client
+
+func GetOrCreateClient(ctx context.Context) (*storage.Client, error) {
+	var err error
+	if gcsClient == nil {
+		once.Do(func() {
+			gcsClient, err = storage.NewClient(ctx)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create storage client: %v", err)
+		}
+		return gcsClient, nil
+	}
+	return gcsClient, nil
+}
diff --git a/accessors/spanner/spanner_accessor.go b/accessors/spanner/spanner_accessor.go
new file mode 100644
index 0000000000..101a9dd93e
--- /dev/null
+++ b/accessors/spanner/spanner_accessor.go
@@ -0,0 +1,203 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spanneracc
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/spanner"
+	"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
+	"cloud.google.com/go/spanner/admin/instance/apiv1/instancepb"
+	spanneradmin "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/admin"
+	spannerclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/client"
+	spinstanceadmin "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/instanceadmin"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	"google.golang.org/api/iterator"
+)
+
+func GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) {
+	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return adminClient.GetDatabase(ctx, &databasepb.GetDatabaseRequest{Name: dbURI})
+}
+
+func GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
+	result, err := GetDatabase(ctx, dbURI)
+	if err != nil {
+		return "", fmt.Errorf("cannot connect to database: %v", err)
+	}
+	return strings.ToLower(result.DatabaseDialect.String()), nil
+}
+
+// CheckExistingDb checks whether the database with dbURI exists or not.
+// If API call doesn't respond then user is informed after every 5 minutes on command line.
+func CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
+	gotResponse := make(chan bool)
+	var err error
+	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
+	if err != nil {
+		return false, err
+	}
+	go func() {
+		_, err = adminClient.GetDatabase(ctx, &databasepb.GetDatabaseRequest{Name: dbURI})
+		gotResponse <- true
+	}()
+	for {
+		select {
+		case <-time.After(5 * time.Minute):
+			fmt.Println("WARNING! API call not responding: make sure that spanner api endpoint is configured properly")
+		case <-gotResponse:
+			if err != nil {
+				if utils.ContainsAny(strings.ToLower(err.Error()), []string{"database not found"}) {
+					return false, nil
+				}
+				return false, fmt.Errorf("can't get database info: %s", err)
+			}
+			return true, nil
+		}
+	}
+}
+
+func CreateEmptyDatabase(ctx context.Context, dbURI string) error {
+	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
+	if err != nil {
+		return err
+	}
+	project, instance, dbName := utils.ParseDbURI(dbURI)
+	req := &databasepb.CreateDatabaseRequest{
+		Parent:          fmt.Sprintf("projects/%s/instances/%s", project, instance),
+		CreateStatement: "CREATE DATABASE `" + dbName + "`",
+	}
+	op, err := adminClient.CreateDatabase(ctx, req)
+	if err != nil {
+		return fmt.Errorf("can't build CreateDatabaseRequest: %w", utils.AnalyzeError(err, dbURI))
+	}
+	if _, err := op.Wait(ctx); err != nil {
+		return fmt.Errorf("createDatabase call failed: %w", utils.AnalyzeError(err, dbURI))
+	}
+	return nil
+}
+
+func GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) {
+	instanceClient, err := spinstanceadmin.GetOrCreateClient(ctx)
+	if err != nil {
+		return "", err
+	}
+	instanceInfo, err := instanceClient.GetInstance(ctx, &instancepb.GetInstanceRequest{Name: instanceURI})
+	if err != nil {
+		return "", err
+	}
+	instanceConfig, err := instanceClient.GetInstanceConfig(ctx, &instancepb.GetInstanceConfigRequest{Name: instanceInfo.Config})
+	if err != nil {
+		return "", err
+
+	}
+	for _, replica := range instanceConfig.Replicas {
+		if replica.DefaultLeaderLocation {
+			return replica.Location, nil
+		}
+	}
+	return "", fmt.Errorf("no leader found for spanner instance %s while trying fetch location", instanceURI)
+}
+
+func CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) {
+	spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI)
+	if err != nil {
+		return false, err
+	}
+	stmt := spanner.Statement{
+		SQL: `SELECT * FROM information_schema.change_streams`,
+	}
+	iter := spClient.Single().Query(ctx, stmt)
+	defer iter.Stop()
+	var cs_catalog, cs_schema, cs_name string
+	var coversAll bool
+	csExists := false
+	for {
+		row, err := iter.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return false, fmt.Errorf("couldn't read row from change_streams table: %w", err)
+		}
+		err = row.Columns(&cs_catalog, &cs_schema, &cs_name, &coversAll)
+		if err != nil {
+			return false, fmt.Errorf("can't scan row from change_streams table: %v", err)
+		}
+		if cs_name == changeStreamName {
+			csExists = true
+			break
+		}
+	}
+	return csExists, nil
+}
+
+func ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error {
+	spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI)
+	if err != nil {
+		return err
+	}
+	// Validate if change stream options are set correctly.
+	stmt := spanner.Statement{
+		SQL: `SELECT option_value FROM information_schema.change_stream_options
+		WHERE change_stream_name = @p1 AND option_name = 'value_capture_type'`,
+		Params: map[string]interface{}{
+			"p1": changeStreamName,
+		},
+	}
+	iter := spClient.Single().Query(ctx, stmt)
+	defer iter.Stop()
+	var option_value string
+	for {
+		row, err := iter.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("couldn't read row from change_stream_options table: %w", err)
+		}
+		err = row.Columns(&option_value)
+		if err != nil {
+			return fmt.Errorf("can't scan row from change_stream_options table: %v", err)
+		}
+		if option_value != "NEW_ROW" {
+			return fmt.Errorf("VALUE_CAPTURE_TYPE for changestream %s is not NEW_ROW. Please update the changestream option or create a new one", changeStreamName)
+		}
+	}
+	return nil
+}
+
+func CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error {
+	spClient, _ := spanneradmin.GetOrCreateClient(ctx)
+	op, err := spClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{
+		Database: dbURI,
+		// TODO: create change stream for only the tables present in Spanner.
+		Statements: []string{fmt.Sprintf("CREATE CHANGE STREAM %s FOR ALL OPTIONS (value_capture_type = 'NEW_ROW')", changeStreamName)},
+	})
+	if err != nil {
+		return fmt.Errorf("cannot submit request create change stream request: %v", err)
+	}
+	if err := op.Wait(ctx); err != nil {
+		return fmt.Errorf("could not update database ddl: %v", err)
+	} else {
+		fmt.Println("Successfully created changestream", changeStreamName)
+	}
+	return nil
+}
diff --git a/accessors/storage/storage_accessor.go b/accessors/storage/storage_accessor.go
new file mode 100644
index 0000000000..0d880b978a
--- /dev/null
+++ b/accessors/storage/storage_accessor.go
@@ -0,0 +1,192 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package storageacc
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	storageclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/storage"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"google.golang.org/api/googleapi"
+)
+
+func CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error {
+	return createGCSBucketUtil(ctx, bucketName, projectID, location, nil, 0)
+}
+
+func CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
+	return createGCSBucketUtil(ctx, bucketName, projectID, location, matchesPrefix, ttl)
+}
+
+func createGCSBucketUtil(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
+	client, err := storageclient.GetOrCreateClient(ctx)
+	if err != nil {
+		return err
+	}
+	bucket := client.Bucket(bucketName)
+	attrs := storage.BucketAttrs{
+		Location: location,
+	}
+	if ttl > 0 {
+		attrs.Lifecycle = storage.Lifecycle{
+			Rules: []storage.LifecycleRule{
+				{
+					Action: storage.LifecycleAction{Type: "Delete"},
+					Condition: storage.LifecycleCondition{
+						AgeInDays: ttl,
+						// The prefixes should not contain the bucket names and starting slash.
+						// For object gs://my_bucket/pictures/paris_2022.jpg,
+						// you would use a condition such as "matchesPrefix":["pictures/paris_"].
+						MatchesPrefix: matchesPrefix,
+					},
+				},
+			},
+		}
+	}
+
+	if err := bucket.Create(ctx, projectID, &attrs); err != nil {
+		if e, ok := err.(*googleapi.Error); ok {
+			// Ignoring the bucket already exists error.
+			if e.Code != 409 {
+				return fmt.Errorf("failed to create bucket: %v", err)
+			} else {
+				fmt.Printf("Using the existing bucket: %v \n", bucketName)
+			}
+		} else {
+			return fmt.Errorf("failed to create bucket: %v", err)
+		}
+
+	} else {
+		fmt.Printf("Created new GCS bucket: %v\n", bucketName)
+	}
+	return nil
+}
+
+// Applies the bucket lifecycle with delete rule. Only accepts the Age and
+// prefix rule conditions as it is only used for the Datastream destination
+// bucket currently.
+func EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error {
+	client, err := storageclient.GetOrCreateClient(ctx)
+	if err != nil {
+		return fmt.Errorf("could not create client while enabling lifecycle: %w", err)
+	}
+
+	for i, str := range matchesPrefix {
+		matchesPrefix[i] = strings.TrimPrefix(str, "/")
+	}
+	bucket := client.Bucket(bucketName)
+	bucketAttrsToUpdate := storage.BucketAttrsToUpdate{
+		Lifecycle: &storage.Lifecycle{
+			Rules: []storage.LifecycleRule{
+				{
+					Action: storage.LifecycleAction{Type: "Delete"},
+					Condition: storage.LifecycleCondition{
+						AgeInDays: ttl,
+						// The prefixes should not contain the bucket names and starting slash.
+						// For object gs://my_bucket/pictures/paris_2022.jpg,
+						// you would use a condition such as "matchesPrefix":["pictures/paris_"].
+						MatchesPrefix: matchesPrefix,
+					},
+				},
+			},
+		},
+	}
+
+	attrs, err := bucket.Update(ctx, bucketAttrsToUpdate)
+	if err != nil {
+		return fmt.Errorf("could not bucket with lifecycle: %w", err)
+	}
+	logger.Log.Info(fmt.Sprintf("Added lifecycle rule to bucket %v\n. Rule Action: %v\t Rule Condition: %v\n",
+		bucketName, attrs.Lifecycle.Rules[0].Action, attrs.Lifecycle.Rules[0].Condition))
+	return nil
+}
+
+// UploadLocalFileToGCS uploads an object.
+func UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error {
+	data, err := os.ReadFile(localFilePath)
+	if err != nil {
+		return fmt.Errorf("could not read file %s: %w", localFilePath, err)
+	}
+	return WriteDataToGCS(ctx, filePath, fileName, string(data))
+}
+
+func WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error {
+	client, err := storageclient.GetOrCreateClient(ctx)
+	if err != nil {
+		return fmt.Errorf("could not create client while uploading to GCS: %w", err)
+	}
+
+	u, err := utils.ParseGCSFilePath(filePath)
+	if err != nil {
+		return fmt.Errorf("parseFilePath: unable to parse file path: %v", err)
+	}
+	bucketName := u.Host
+	bucket := client.Bucket(bucketName)
+	obj := bucket.Object(u.Path[1:] + fileName)
+
+	w := obj.NewWriter(ctx)
+	if _, err := fmt.Fprint(w, data); err != nil {
+		fmt.Printf("Failed to write to Cloud Storage: %s", filePath)
+		return err
+	}
+	if err := w.Close(); err != nil {
+		fmt.Printf("Failed to close GCS file: %s", filePath)
+		return err
+	}
+	return nil
+}
+
+func ReadGcsFile(ctx context.Context, filePath string) (string, error) {
+	client, err := storageclient.GetOrCreateClient(ctx)
+	if err != nil {
+		return "", fmt.Errorf("could not create client: %w", err)
+	}
+
+	u, err := utils.ParseGCSFilePath(filePath)
+	if err != nil {
+		return "", fmt.Errorf("unable to parse file path: %v", err)
+	}
+	bucketName := u.Host
+	bucket := client.Bucket(bucketName)
+	obj := bucket.Object(u.Path[1:])
+
+	rc, err := obj.NewReader(ctx)
+	if err != nil {
+		return "", err
+	}
+	defer rc.Close()
+	buf := new(strings.Builder)
+	if _, err := io.Copy(buf, rc); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func ReadAnyFile(ctx context.Context, filePath string) (string, error) {
+	if strings.HasPrefix(filePath, constants.GCS_FILE_PREFIX) {
+		return ReadGcsFile(ctx, filePath)
+	}
+	buf, err := os.ReadFile(filePath)
+	if err != nil {
+		return "", err
+	}
+	return string(buf), nil
+}
diff --git a/cmd/data.go b/cmd/data.go
index c908f8ad94..6190ed3696 100644
--- a/cmd/data.go
+++ b/cmd/data.go
@@ -26,6 +26,7 @@ import (
 
 	sp "cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
+	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
@@ -178,7 +179,7 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface
 
 // validateExistingDb validates that the existing spanner schema is in accordance with the one specified in the session file.
 func validateExistingDb(ctx context.Context, spDialect, dbURI string, adminClient *database.DatabaseAdminClient, client *sp.Client, conv *internal.Conv) error {
-	dbExists, err := conversion.CheckExistingDb(ctx, adminClient, dbURI)
+	dbExists, err := spanneracc.CheckExistingDb(ctx, dbURI)
 	if err != nil {
 		err = fmt.Errorf("can't verify target database: %v", err)
 		return err
diff --git a/common/utils/storage_utils.go b/common/utils/storage_utils.go
new file mode 100644
index 0000000000..3429c6d293
--- /dev/null
+++ b/common/utils/storage_utils.go
@@ -0,0 +1,41 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package utils contains common helper functions used across multiple other packages.
+// Utils should not import any Spanner migration tool packages.
+package utils
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+)
+
+func ParseGCSFilePath(filePath string) (*url.URL, error) {
+	if len(filePath) == 0 {
+		return nil, fmt.Errorf("found empty GCS path")
+	}
+	if filePath[len(filePath)-1] != '/' {
+		filePath = filePath + "/"
+	}
+	u, err := url.Parse(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("parseFilePath: unable to parse file path %s", filePath)
+	}
+	if u.Scheme != constants.GCS_SCHEME {
+		return nil, fmt.Errorf("not a valid GCS path: %s, should start with 'gs'", filePath)
+	}
+	return u, nil
+}
diff --git a/common/utils/utils.go b/common/utils/utils.go
index 70ec53f882..932da34f11 100644
--- a/common/utils/utils.go
+++ b/common/utils/utils.go
@@ -19,11 +19,11 @@ package utils
 import (
 	"bufio"
 	"context"
+	"crypto/rand"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"net/url"
 	"os"
 	"os/exec"
@@ -37,6 +37,7 @@ import (
 	sp "cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
 	instance "cloud.google.com/go/spanner/admin/instance/apiv1"
+	"cloud.google.com/go/spanner/admin/instance/apiv1/instancepb"
 	"cloud.google.com/go/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
@@ -44,10 +45,8 @@ import (
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/sources/spanner"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl"
 	"golang.org/x/crypto/ssh/terminal"
-	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
-	instancepb "google.golang.org/genproto/googleapis/spanner/admin/instance/v1"
 )
 
 // IOStreams is a struct that contains the file descriptor for dumpFile.
@@ -173,82 +172,6 @@ func PreloadGCSFiles(tables []ManifestTable) ([]ManifestTable, error) {
 	return tables, nil
 }
 
-func ParseGCSFilePath(filePath string) (*url.URL, error) {
-	if len(filePath) == 0 {
-		return nil, fmt.Errorf("found empty GCS path")
-	}
-	if filePath[len(filePath)-1] != '/' {
-		filePath = filePath + "/"
-	}
-	u, err := url.Parse(filePath)
-	if err != nil {
-		return nil, fmt.Errorf("parseFilePath: unable to parse file path %s", filePath)
-	}
-	if u.Scheme != constants.GCS_SCHEME {
-		return nil, fmt.Errorf("not a valid GCS path: %s, should start with 'gs'", filePath)
-	}
-	return u, nil
-}
-
-func WriteToGCS(filePath, fileName, data string) error {
-	ctx := context.Background()
-
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		fmt.Printf("Failed to create GCS client")
-		return err
-	}
-	defer client.Close()
-	u, err := ParseGCSFilePath(filePath)
-	if err != nil {
-		return fmt.Errorf("parseFilePath: unable to parse file path: %v", err)
-	}
-	bucketName := u.Host
-	bucket := client.Bucket(bucketName)
-	obj := bucket.Object(u.Path[1:] + fileName)
-
-	w := obj.NewWriter(ctx)
-	if _, err := fmt.Fprint(w, data); err != nil {
-		fmt.Printf("Failed to write to Cloud Storage: %s", filePath)
-		return err
-	}
-	if err := w.Close(); err != nil {
-		fmt.Printf("Failed to close GCS file: %s", filePath)
-		return err
-	}
-	return nil
-}
-
-func CreateGCSBucket(bucketName, projectID, location string) error {
-	ctx := context.Background()
-
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to create GCS client: %v", err)
-	}
-	defer client.Close()
-	bucket := client.Bucket(bucketName)
-	attrs := storage.BucketAttrs{
-		Location: location,
-	}
-	if err := bucket.Create(ctx, projectID, &attrs); err != nil {
-		if e, ok := err.(*googleapi.Error); ok {
-			// Ignoring the bucket already exists error.
-			if e.Code != 409 {
-				return fmt.Errorf("failed to create bucket: %v", err)
-			} else {
-				fmt.Printf("Using the existing bucket: %v \n", bucketName)
-			}
-		} else {
-			return fmt.Errorf("failed to create bucket: %v", err)
-		}
-
-	} else {
-		fmt.Printf("Created new GCS bucket: %v\n", bucketName)
-	}
-	return nil
-}
-
 // GetProject returns the cloud project we should use for accessing Spanner.
 // Use environment variable GCLOUD_PROJECT if it is set.
 // Otherwise, use the default project returned from gcloud.
@@ -347,6 +270,12 @@ func GenerateName(prefix string) (string, error) {
 	return fmt.Sprintf("%s_%x-%x", prefix, b[0:2], b[2:4]), nil
 }
 
+func GenerateHashStr() string {
+	b := make([]byte, 4)
+	rand.Read(b)
+	return fmt.Sprintf("%x-%x", b[0:2], b[2:4])
+}
+
 // parseURI parses an unknown URI string that could be a database, instance or project URI.
 func parseURI(URI string) (project, instance, dbName string) {
 	project, instance, dbName = "", "", ""
diff --git a/conversion/conversion.go b/conversion/conversion.go
index 20216dfea4..1608f98054 100644
--- a/conversion/conversion.go
+++ b/conversion/conversion.go
@@ -44,6 +44,8 @@ import (
 	datastream "cloud.google.com/go/datastream/apiv1"
 	sp "cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
+	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/metrics"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -355,7 +357,7 @@ func dataFromDatabase(ctx context.Context, sourceProfile profiles.SourceProfile,
 			// Try to apply lifecycle rule to Datastream destination bucket.
 			gcsConfig := streamingCfg.GcsCfg
 			if gcsConfig.TtlInDaysSet {
-				err = streaming.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
+				err = storageacc.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
 				if err != nil {
 					logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err))
 					logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n")
@@ -476,7 +478,7 @@ func dataFromDatabaseForDataflowMigration(targetProfile profiles.TargetProfile,
 		// Try to apply lifecycle rule to Datastream destination bucket.
 		gcsConfig := streamingCfg.GcsCfg
 		if gcsConfig.TtlInDaysSet {
-			err = streaming.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
+			err = storageacc.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
 			if err != nil {
 				logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err))
 				logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n")
@@ -520,11 +522,11 @@ func dataFromDatabaseForDataflowMigration(targetProfile profiles.TargetProfile,
 
 	// create monitoring aggregated dashboard for sharded migration
 	aggMonitoringResources := metrics.MonitoringMetricsResources{
-		ProjectId:                     targetProfile.Conn.Sp.Project,
-		SpannerInstanceId:             targetProfile.Conn.Sp.Instance,
-		SpannerDatabaseId:             targetProfile.Conn.Sp.Dbname,
-		ShardToShardResourcesMap:      conv.Audit.StreamingStats.ShardToShardResourcesMap,
-		MigrationRequestId:            conv.Audit.MigrationRequestId,
+		ProjectId:                targetProfile.Conn.Sp.Project,
+		SpannerInstanceId:        targetProfile.Conn.Sp.Instance,
+		SpannerDatabaseId:        targetProfile.Conn.Sp.Dbname,
+		ShardToShardResourcesMap: conv.Audit.StreamingStats.ShardToShardResourcesMap,
+		MigrationRequestId:       conv.Audit.MigrationRequestId,
 	}
 	aggRespDash, dashboardErr := aggMonitoringResources.CreateDataflowAggMonitoringDashboard(ctx)
 	if dashboardErr != nil {
@@ -798,7 +800,7 @@ func getSeekable(f *os.File) (*os.File, int64, error) {
 
 // VerifyDb checks whether the db exists and if it does, verifies if the schema is what we currently support.
 func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (dbExists bool, err error) {
-	dbExists, err = CheckExistingDb(ctx, adminClient, dbURI)
+	dbExists, err = spanneracc.CheckExistingDb(ctx, dbURI)
 	if err != nil {
 		return dbExists, err
 	}
@@ -808,31 +810,6 @@ func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, db
 	return dbExists, err
 }
 
-// CheckExistingDb checks whether the database with dbURI exists or not.
-// If API call doesn't respond then user is informed after every 5 minutes on command line.
-func CheckExistingDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (bool, error) {
-	gotResponse := make(chan bool)
-	var err error
-	go func() {
-		_, err = adminClient.GetDatabase(ctx, &adminpb.GetDatabaseRequest{Name: dbURI})
-		gotResponse <- true
-	}()
-	for {
-		select {
-		case <-time.After(5 * time.Minute):
-			fmt.Println("WARNING! API call not responding: make sure that spanner api endpoint is configured properly")
-		case <-gotResponse:
-			if err != nil {
-				if utils.ContainsAny(strings.ToLower(err.Error()), []string{"database not found"}) {
-					return false, nil
-				}
-				return false, fmt.Errorf("can't get database info: %s", err)
-			}
-			return true, nil
-		}
-	}
-}
-
 // ValidateTables validates that all the tables in the database are empty.
 // It returns the name of the first non-empty table if found, and an empty string otherwise.
 func ValidateTables(ctx context.Context, client *sp.Client, spDialect string) (string, error) {
diff --git a/streaming/streaming.go b/streaming/streaming.go
index 1cf1f21655..cfea9fcd22 100644
--- a/streaming/streaming.go
+++ b/streaming/streaming.go
@@ -33,11 +33,13 @@ import (
 
 	resourcemanager "cloud.google.com/go/resourcemanager/apiv3"
 	resourcemanagerpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
+	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/profiles"
 	"github.com/google/uuid"
 	"github.com/googleapis/gax-go/v2"
@@ -863,7 +865,7 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("can't encode session state to JSON: %v", err)
 	}
-	err = utils.WriteToGCS(streamingCfg.TmpDir, "session.json", string(convJSON))
+	err = storageacc.WriteDataToGCS(ctx, streamingCfg.TmpDir, "session.json", string(convJSON))
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err)
 	}
@@ -874,7 +876,7 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("failed to compute transformation context: %s", err.Error())
 	}
-	err = utils.WriteToGCS(streamingCfg.TmpDir, "transformationContext.json", string(transformationContext))
+	err = storageacc.WriteDataToGCS(ctx, streamingCfg.TmpDir, "transformationContext.json", string(transformationContext))
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err)
 	}
@@ -884,43 +886,3 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st
 	}
 	return dfOutput, nil
 }
-
-// Applies the bucket lifecycle with delete rule. Only accepts the Age and
-// prefix rule conditions as it is only used for the Datastream destination
-// bucket currently.
-func EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error {
-	client, err := storage.NewClient(ctx)
-	if err != nil {
-		return fmt.Errorf("could not create client while enabling lifecycle: %w", err)
-	}
-	defer client.Close()
-
-	for i, str := range matchesPrefix {
-		matchesPrefix[i] = strings.TrimPrefix(str, "/")
-	}
-	bucket := client.Bucket(bucketName)
-	bucketAttrsToUpdate := storage.BucketAttrsToUpdate{
-		Lifecycle: &storage.Lifecycle{
-			Rules: []storage.LifecycleRule{
-				{
-					Action: storage.LifecycleAction{Type: "Delete"},
-					Condition: storage.LifecycleCondition{
-						AgeInDays: ttl,
-						// The prefixes should not contain the bucket names and starting slash.
-						// For object gs://my_bucket/pictures/paris_2022.jpg,
-						// you would use a condition such as "matchesPrefix":["pictures/paris_"].
-						MatchesPrefix: matchesPrefix,
-					},
-				},
-			},
-		},
-	}
-
-	attrs, err := bucket.Update(ctx, bucketAttrsToUpdate)
-	if err != nil {
-		return fmt.Errorf("could not bucket with lifecycle: %w", err)
-	}
-	logger.Log.Info(fmt.Sprintf("Added lifecycle rule to bucket %v\n. Rule Action: %v\t Rule Condition: %v\n",
-		bucketName, attrs.Lifecycle.Rules[0].Action, attrs.Lifecycle.Rules[0].Condition))
-	return nil
-}
diff --git a/testing/accessors/spanner/spanner_accessor_test.go b/testing/accessors/spanner/spanner_accessor_test.go
new file mode 100644
index 0000000000..1d7e77bd72
--- /dev/null
+++ b/testing/accessors/spanner/spanner_accessor_test.go
@@ -0,0 +1,129 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// TODO: Refactor this file and other integration tests by moving all common code
+// to remove redundancy.
+
+package utils_test
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	database "cloud.google.com/go/spanner/admin/database/apiv1"
+	"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
+	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+)
+
+var (
+	projectID  string
+	instanceID string
+
+	ctx           context.Context
+	databaseAdmin *database.DatabaseAdminClient
+)
+
+func TestMain(m *testing.M) {
+	cleanup := initTests()
+	res := m.Run()
+	cleanup()
+	os.Exit(res)
+}
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func initTests() (cleanup func()) {
+	projectID = os.Getenv("SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_PROJECT_ID")
+	instanceID = os.Getenv("SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_INSTANCE_ID")
+
+	ctx = context.Background()
+	flag.Parse() // Needed for testing.Short().
+	noop := func() {}
+
+	if testing.Short() {
+		log.Println("Unit test for UpdateDDLForeignKeys skipped in -short mode.")
+		return noop
+	}
+
+	if projectID == "" {
+		log.Println("Unit test for UpdateDDLForeignKeys skipped: SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_PROJECT_ID is missing")
+		return noop
+	}
+
+	if instanceID == "" {
+		log.Println("Unit test for UpdateDDLForeignKeys skipped: SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_INSTANCE_ID is missing")
+		return noop
+	}
+
+	var err error
+	databaseAdmin, err = database.NewDatabaseAdminClient(ctx)
+	if err != nil {
+		log.Fatalf("cannot create databaseAdmin client: %v", err)
+	}
+
+	return func() {
+		databaseAdmin.Close()
+	}
+}
+
+func dropDatabase(t *testing.T, dbPath string) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	// Drop the testing database.
+	if err := databaseAdmin.DropDatabase(ctx, &databasepb.DropDatabaseRequest{Database: dbPath}); err != nil {
+		t.Fatalf("failed to drop testing database %v: %v", dbPath, err)
+	}
+}
+
+func TestCheckExistingDb(t *testing.T) {
+	onlyRunForEmulatorTest(t)
+	dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, "check-db-exists")
+	err := conversion.CreateDatabase(ctx, databaseAdmin, dbURI, internal.MakeConv(), os.Stdout, "", constants.BULK_MIGRATION)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dropDatabase(t, dbURI)
+	testCases := []struct {
+		dbName   string
+		dbExists bool
+	}{
+		{"check-db-exists", true},
+		{"check-db-does-not-exist", false},
+	}
+
+	for _, tc := range testCases {
+		dbExists, err := spanneracc.CheckExistingDb(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName))
+		assert.Nil(t, err)
+		assert.Equal(t, tc.dbExists, dbExists)
+	}
+}
+
+func onlyRunForEmulatorTest(t *testing.T) {
+	if os.Getenv("SPANNER_EMULATOR_HOST") == "" {
+		t.Skip("Skipping tests only running against the emulator.")
+	}
+}
diff --git a/testing/conversion/conversion_test.go b/testing/conversion/conversion_test.go
index ff9a8e49ef..ba08519a71 100644
--- a/testing/conversion/conversion_test.go
+++ b/testing/conversion/conversion_test.go
@@ -248,29 +248,6 @@ func TestVerifyDb(t *testing.T) {
 	}
 }
 
-func TestCheckExistingDb(t *testing.T) {
-	onlyRunForEmulatorTest(t)
-	dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, "check-db-exists")
-	err := conversion.CreateDatabase(ctx, databaseAdmin, dbURI, internal.MakeConv(), os.Stdout, "", constants.BULK_MIGRATION)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer dropDatabase(t, dbURI)
-	testCases := []struct {
-		dbName   string
-		dbExists bool
-	}{
-		{"check-db-exists", true},
-		{"check-db-does-not-exist", false},
-	}
-
-	for _, tc := range testCases {
-		dbExists, err := conversion.CheckExistingDb(ctx, databaseAdmin, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName))
-		assert.Nil(t, err)
-		assert.Equal(t, tc.dbExists, dbExists)
-	}
-}
-
 func TestValidateDDL(t *testing.T) {
 	onlyRunForEmulatorTest(t)
 
diff --git a/webv2/helpers/helpers.go b/webv2/helpers/helpers.go
index d0dbea31f9..7495d9324a 100644
--- a/webv2/helpers/helpers.go
+++ b/webv2/helpers/helpers.go
@@ -21,8 +21,8 @@ import (
 	"strings"
 
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
+	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
-	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
 	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
 )
 
@@ -160,7 +160,7 @@ func CheckOrCreateMetadataDb(projectId string, instanceId string) bool {
 	}
 	defer adminClient.Close()
 
-	dbExists, err := conversion.CheckExistingDb(ctx, adminClient, uri)
+	dbExists, err := spanneracc.CheckExistingDb(ctx, uri)
 	if err != nil {
 		fmt.Println(err)
 		return false
diff --git a/webv2/profile/profile.go b/webv2/profile/profile.go
index 9808a594f6..e46a78ed34 100644
--- a/webv2/profile/profile.go
+++ b/webv2/profile/profile.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	datastream "cloud.google.com/go/datastream/apiv1"
+	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/streaming"
@@ -160,7 +161,7 @@ func CreateConnectionProfile(w http.ResponseWriter, r *http.Request) {
 		} else {
 			bucketName = strings.ToLower(sessionState.Conv.Audit.MigrationRequestId)
 		}
-		err = utils.CreateGCSBucket(bucketName, sessionState.GCPProjectID, sessionState.Region)
+		err = storageacc.CreateGCSBucket(ctx, bucketName, sessionState.GCPProjectID, sessionState.Region)
 		if err != nil {
 			http.Error(w, fmt.Sprintf("Error while creating bucket: %v", err), http.StatusBadRequest)
 			return
diff --git a/webv2/session/session_service.go b/webv2/session/session_service.go
index df190eac4c..d6a26f8685 100644
--- a/webv2/session/session_service.go
+++ b/webv2/session/session_service.go
@@ -7,7 +7,7 @@ import (
 	"cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
 	"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
-	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
+	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	helpers "github.com/GoogleCloudPlatform/spanner-migration-tool/webv2/helpers"
 )
 
@@ -88,7 +88,7 @@ func migrateMetadataDb(projectId, instanceId string) {
 	defer adminClient.Close()
 
 	oldMetadataDbUri := getOldMetadataDbUri(projectId, instanceId)
-	oldMetadataDBExists, err := conversion.CheckExistingDb(ctx, adminClient, oldMetadataDbUri)
+	oldMetadataDBExists, err := spanneracc.CheckExistingDb(ctx, oldMetadataDbUri)
 	if err != nil {
 		fmt.Printf("could not check if oldMetadataDB exists. error=%v\n", err)
 		return
diff --git a/webv2/web.go b/webv2/web.go
index 7b8c84b8cc..0ae24e2f33 100644
--- a/webv2/web.go
+++ b/webv2/web.go
@@ -36,6 +36,7 @@ import (
 	"time"
 
 	instance "cloud.google.com/go/spanner/admin/instance/apiv1"
+	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/cmd"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -2263,7 +2264,7 @@ func migrate(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, fmt.Sprintf("Can't get source and target profiles: %v", err), http.StatusBadRequest)
 		return
 	}
-	err = writeSessionFile(sessionState)
+	err = writeSessionFile(ctx, sessionState)
 	if err != nil {
 		log.Println("can't write session file")
 		http.Error(w, fmt.Sprintf("Can't write session file to GCS: %v", err), http.StatusBadRequest)
@@ -2484,9 +2485,9 @@ func createConfigFileForShardedBulkMigration(sessionState *session.SessionState,
 	return nil
 }
 
-func writeSessionFile(sessionState *session.SessionState) error {
+func writeSessionFile(ctx context.Context, sessionState *session.SessionState) error {
 
-	err := utils.CreateGCSBucket(sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region)
+	err := storageacc.CreateGCSBucket(ctx, sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region)
 	if err != nil {
 		return fmt.Errorf("error while creating bucket: %v", err)
 	}
@@ -2495,7 +2496,7 @@ func writeSessionFile(sessionState *session.SessionState) error {
 	if err != nil {
 		return fmt.Errorf("can't encode session state to JSON: %v", err)
 	}
-	err = utils.WriteToGCS("gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON))
+	err = storageacc.WriteDataToGCS(ctx, "gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON))
 	if err != nil {
 		return fmt.Errorf("error while writing to GCS: %v", err)
 	}
@@ -3031,7 +3032,7 @@ type ResourceDetails struct {
 	ResourceType string `json:"ResourceType"`
 	ResourceName string `json:"ResourceName"`
 	ResourceUrl  string `json:"ResourceUrl"`
-	GcloudCmd string `json:"GcloudCmd"`
+	GcloudCmd    string `json:"GcloudCmd"`
 }
 type GeneratedResources struct {
 	MigrationJobId string `json:"MigrationJobId"`
@@ -3054,7 +3055,7 @@ type GeneratedResources struct {
 	AggMonitoringDashboardName string `json:"AggMonitoringDashboardName"`
 	AggMonitoringDashboardUrl  string `json:"AggMonitoringDashboardUrl"`
 	//Used for sharded migration flow
-	ShardToShardResourcesMap	  map[string][]ResourceDetails `json:"ShardToShardResourcesMap"`
+	ShardToShardResourcesMap map[string][]ResourceDetails `json:"ShardToShardResourcesMap"`
 }
 
 func addTypeToList(convertedType string, spType string, issues []internal.SchemaIssue, l []typeIssue) []typeIssue {

From f6530b1bf784662d84627997e9fb5855a50c6f17 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:45:55 +0530
Subject: [PATCH 16/35] Add Unmarshall method

---
 accessors/dataflow/dataflow_accessor.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 3cffedaaba..12b0962ee2 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -15,10 +15,12 @@ package dataflowaccessor
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/dataflow"
+	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"github.com/googleapis/gax-go/v2"
 )
@@ -41,3 +43,16 @@ func (dfA *DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *da
 	}
 	return respDf, nil
 }
+
+func UnmarshalDataflowTuningConfig(ctx context.Context, filePath string) (DataflowTuningConfig, error) {
+	jsonStr, err := storageacc.ReadAnyFile(ctx, filePath)
+	if err != nil {
+		return DataflowTuningConfig{}, err
+	}
+	tuningCfg := DataflowTuningConfig{}
+	err = json.Unmarshal([]byte(jsonStr), &tuningCfg)
+	if err != nil {
+		return DataflowTuningConfig{}, err
+	}
+	return tuningCfg, nil
+}

From 0c4d854b8d71f2d2199387c16ed469b4e041cdfc Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 22:45:17 +0530
Subject: [PATCH 17/35] Rename storageacc and spanner acc to storageaccessor
 and spanneraccessor

---
 accessors/dataflow/dataflow_accessor.go       | 15 ----
 accessors/spanner/spanner_accessor.go         |  2 +-
 accessors/storage/storage_accessor.go         |  2 +-
 accessors/utils/dataflow/dataflow_utils.go    | 16 +++++
 cmd/data.go                                   |  4 +-
 common/constants/constants.go                 |  3 +-
 conversion/conversion.go                      | 72 +++++++++----------
 streaming/streaming.go                        |  6 +-
 .../spanner/spanner_accessor_test.go          |  6 +-
 webv2/helpers/helpers.go                      |  4 +-
 webv2/profile/profile.go                      |  4 +-
 webv2/session/session_service.go              |  4 +-
 webv2/web.go                                  |  6 +-
 13 files changed, 73 insertions(+), 71 deletions(-)

diff --git a/accessors/dataflow/dataflow_accessor.go b/accessors/dataflow/dataflow_accessor.go
index 12b0962ee2..3cffedaaba 100644
--- a/accessors/dataflow/dataflow_accessor.go
+++ b/accessors/dataflow/dataflow_accessor.go
@@ -15,12 +15,10 @@ package dataflowaccessor
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/dataflow"
-	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"github.com/googleapis/gax-go/v2"
 )
@@ -43,16 +41,3 @@ func (dfA *DataflowAccessorImpl) LaunchFlexTemplate(ctx context.Context, req *da
 	}
 	return respDf, nil
 }
-
-func UnmarshalDataflowTuningConfig(ctx context.Context, filePath string) (DataflowTuningConfig, error) {
-	jsonStr, err := storageacc.ReadAnyFile(ctx, filePath)
-	if err != nil {
-		return DataflowTuningConfig{}, err
-	}
-	tuningCfg := DataflowTuningConfig{}
-	err = json.Unmarshal([]byte(jsonStr), &tuningCfg)
-	if err != nil {
-		return DataflowTuningConfig{}, err
-	}
-	return tuningCfg, nil
-}
diff --git a/accessors/spanner/spanner_accessor.go b/accessors/spanner/spanner_accessor.go
index 101a9dd93e..1b1594c1e6 100644
--- a/accessors/spanner/spanner_accessor.go
+++ b/accessors/spanner/spanner_accessor.go
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package spanneracc
+package spanneraccessor
 
 import (
 	"context"
diff --git a/accessors/storage/storage_accessor.go b/accessors/storage/storage_accessor.go
index 0d880b978a..f587b32627 100644
--- a/accessors/storage/storage_accessor.go
+++ b/accessors/storage/storage_accessor.go
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package storageacc
+package storageaccessor
 
 import (
 	"context"
diff --git a/accessors/utils/dataflow/dataflow_utils.go b/accessors/utils/dataflow/dataflow_utils.go
index 335fe8ed0d..8266f720ac 100644
--- a/accessors/utils/dataflow/dataflow_utils.go
+++ b/accessors/utils/dataflow/dataflow_utils.go
@@ -14,12 +14,15 @@
 package dataflowutils
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"golang.org/x/exp/maps"
 )
@@ -153,3 +156,16 @@ func formatAdditionalUserLabels(labels map[string]string) string {
 	}
 	return strings.Join(res, ",")
 }
+
+func UnmarshalDataflowTuningConfig(ctx context.Context, filePath string) (dataflowaccessor.DataflowTuningConfig, error) {
+	jsonStr, err := storageaccessor.ReadAnyFile(ctx, filePath)
+	if err != nil {
+		return dataflowaccessor.DataflowTuningConfig{}, err
+	}
+	tuningCfg := dataflowaccessor.DataflowTuningConfig{}
+	err = json.Unmarshal([]byte(jsonStr), &tuningCfg)
+	if err != nil {
+		return dataflowaccessor.DataflowTuningConfig{}, err
+	}
+	return tuningCfg, nil
+}
diff --git a/cmd/data.go b/cmd/data.go
index 6190ed3696..c739859bf7 100644
--- a/cmd/data.go
+++ b/cmd/data.go
@@ -26,7 +26,7 @@ import (
 
 	sp "cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
-	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
@@ -179,7 +179,7 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface
 
 // validateExistingDb validates that the existing spanner schema is in accordance with the one specified in the session file.
 func validateExistingDb(ctx context.Context, spDialect, dbURI string, adminClient *database.DatabaseAdminClient, client *sp.Client, conv *internal.Conv) error {
-	dbExists, err := spanneracc.CheckExistingDb(ctx, dbURI)
+	dbExists, err := spanneraccessor.CheckExistingDb(ctx, dbURI)
 	if err != nil {
 		err = fmt.Errorf("can't verify target database: %v", err)
 		return err
diff --git a/common/constants/constants.go b/common/constants/constants.go
index e98855a81d..c0eaa87685 100644
--- a/common/constants/constants.go
+++ b/common/constants/constants.go
@@ -64,7 +64,8 @@ const (
 	MigrationMetadataKey string = "cloud-spanner-migration-metadata"
 
 	// Scheme used for GCS paths
-	GCS_SCHEME string = "gs"
+	GCS_SCHEME      string = "gs"
+	GCS_FILE_PREFIX string = "gs://"
 
 	// File upload prefix for dump and session load.
 	UPLOAD_FILE_DIR string = "upload-file"
diff --git a/conversion/conversion.go b/conversion/conversion.go
index 1608f98054..c6376f0215 100644
--- a/conversion/conversion.go
+++ b/conversion/conversion.go
@@ -44,8 +44,8 @@ import (
 	datastream "cloud.google.com/go/datastream/apiv1"
 	sp "cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
-	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
-	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/metrics"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -69,12 +69,12 @@ import (
 	dydb "github.com/aws/aws-sdk-go/service/dynamodb"
 	"github.com/aws/aws-sdk-go/service/dynamodbstreams"
 	mysqldriver "github.com/go-sql-driver/mysql"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/stdlib"
 	"go.uber.org/zap"
 	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/proto"
-	"github.com/jackc/pgx/v5"
-    "github.com/jackc/pgx/v5/stdlib"
 )
 
 var (
@@ -357,7 +357,7 @@ func dataFromDatabase(ctx context.Context, sourceProfile profiles.SourceProfile,
 			// Try to apply lifecycle rule to Datastream destination bucket.
 			gcsConfig := streamingCfg.GcsCfg
 			if gcsConfig.TtlInDaysSet {
-				err = storageacc.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
+				err = storageaccessor.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
 				if err != nil {
 					logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err))
 					logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n")
@@ -478,7 +478,7 @@ func dataFromDatabaseForDataflowMigration(targetProfile profiles.TargetProfile,
 		// Try to apply lifecycle rule to Datastream destination bucket.
 		gcsConfig := streamingCfg.GcsCfg
 		if gcsConfig.TtlInDaysSet {
-			err = storageacc.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
+			err = storageaccessor.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
 			if err != nil {
 				logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err))
 				logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n")
@@ -800,7 +800,7 @@ func getSeekable(f *os.File) (*os.File, int64, error) {
 
 // VerifyDb checks whether the db exists and if it does, verifies if the schema is what we currently support.
 func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (dbExists bool, err error) {
-	dbExists, err = spanneracc.CheckExistingDb(ctx, dbURI)
+	dbExists, err = spanneraccessor.CheckExistingDb(ctx, dbURI)
 	if err != nil {
 		return dbExists, err
 	}
@@ -1282,20 +1282,20 @@ func GetInfoSchemaFromCloudSQL(sourceProfile profiles.SourceProfile, targetProfi
 	switch driver {
 	case constants.MYSQL:
 		d, err := cloudsqlconn.NewDialer(context.Background(), cloudsqlconn.WithIAMAuthN())
-        if err != nil {
-                return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err)
-        }
-        var opts []cloudsqlconn.DialOption
+		if err != nil {
+			return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err)
+		}
+		var opts []cloudsqlconn.DialOption
 		instanceName := fmt.Sprintf("%s:%s:%s", sourceProfile.ConnCloudSQL.Mysql.Project, sourceProfile.ConnCloudSQL.Mysql.Region, sourceProfile.ConnCloudSQL.Mysql.InstanceName)
-        mysqldriver.RegisterDialContext("cloudsqlconn",
-                func(ctx context.Context, addr string) (net.Conn, error) {
-                        return d.Dial(ctx, instanceName, opts...)
-                })
+		mysqldriver.RegisterDialContext("cloudsqlconn",
+			func(ctx context.Context, addr string) (net.Conn, error) {
+				return d.Dial(ctx, instanceName, opts...)
+			})
 
-        dbURI := fmt.Sprintf("%s:empty@cloudsqlconn(localhost:3306)/%s?parseTime=true",
-                sourceProfile.ConnCloudSQL.Mysql.User, sourceProfile.ConnCloudSQL.Mysql.Db)
+		dbURI := fmt.Sprintf("%s:empty@cloudsqlconn(localhost:3306)/%s?parseTime=true",
+			sourceProfile.ConnCloudSQL.Mysql.User, sourceProfile.ConnCloudSQL.Mysql.Db)
 
-        db, err := sql.Open("mysql", dbURI)
+		db, err := sql.Open("mysql", dbURI)
 		if err != nil {
 			return nil, fmt.Errorf("sql.Open: %w", err)
 		}
@@ -1307,25 +1307,25 @@ func GetInfoSchemaFromCloudSQL(sourceProfile profiles.SourceProfile, targetProfi
 		}, nil
 	case constants.POSTGRES:
 		d, err := cloudsqlconn.NewDialer(context.Background(), cloudsqlconn.WithIAMAuthN())
-        if err != nil {
-                return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err)
-        }
-        var opts []cloudsqlconn.DialOption
-
-        dsn := fmt.Sprintf("user=%s database=%s", sourceProfile.ConnCloudSQL.Pg.User, sourceProfile.ConnCloudSQL.Pg.Db)
-        config, err := pgx.ParseConfig(dsn)
-        if err != nil {
-                return nil, err
-        }
+		if err != nil {
+			return nil, fmt.Errorf("cloudsqlconn.NewDialer: %w", err)
+		}
+		var opts []cloudsqlconn.DialOption
+
+		dsn := fmt.Sprintf("user=%s database=%s", sourceProfile.ConnCloudSQL.Pg.User, sourceProfile.ConnCloudSQL.Pg.Db)
+		config, err := pgx.ParseConfig(dsn)
+		if err != nil {
+			return nil, err
+		}
 		instanceName := fmt.Sprintf("%s:%s:%s", sourceProfile.ConnCloudSQL.Pg.Project, sourceProfile.ConnCloudSQL.Pg.Region, sourceProfile.ConnCloudSQL.Pg.InstanceName)
-        config.DialFunc = func(ctx context.Context, network, instance string) (net.Conn, error) {
-                return d.Dial(ctx, instanceName, opts...)
-        }
-        dbURI := stdlib.RegisterConnConfig(config)
-        db, err := sql.Open("pgx", dbURI)
-        if err != nil {
-                return nil, fmt.Errorf("sql.Open: %w", err)
-        }
+		config.DialFunc = func(ctx context.Context, network, instance string) (net.Conn, error) {
+			return d.Dial(ctx, instanceName, opts...)
+		}
+		dbURI := stdlib.RegisterConnConfig(config)
+		db, err := sql.Open("pgx", dbURI)
+		if err != nil {
+			return nil, fmt.Errorf("sql.Open: %w", err)
+		}
 		temp := false
 		return postgres.InfoSchemaImpl{
 			Db:             db,
diff --git a/streaming/streaming.go b/streaming/streaming.go
index cfea9fcd22..57775356a2 100644
--- a/streaming/streaming.go
+++ b/streaming/streaming.go
@@ -33,7 +33,7 @@ import (
 
 	resourcemanager "cloud.google.com/go/resourcemanager/apiv3"
 	resourcemanagerpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
-	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -865,7 +865,7 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("can't encode session state to JSON: %v", err)
 	}
-	err = storageacc.WriteDataToGCS(ctx, streamingCfg.TmpDir, "session.json", string(convJSON))
+	err = storageaccessor.WriteDataToGCS(ctx, streamingCfg.TmpDir, "session.json", string(convJSON))
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err)
 	}
@@ -876,7 +876,7 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("failed to compute transformation context: %s", err.Error())
 	}
-	err = storageacc.WriteDataToGCS(ctx, streamingCfg.TmpDir, "transformationContext.json", string(transformationContext))
+	err = storageaccessor.WriteDataToGCS(ctx, streamingCfg.TmpDir, "transformationContext.json", string(transformationContext))
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err)
 	}
diff --git a/testing/accessors/spanner/spanner_accessor_test.go b/testing/accessors/spanner/spanner_accessor_test.go
index 1d7e77bd72..eb7775fcca 100644
--- a/testing/accessors/spanner/spanner_accessor_test.go
+++ b/testing/accessors/spanner/spanner_accessor_test.go
@@ -15,7 +15,7 @@
 // TODO: Refactor this file and other integration tests by moving all common code
 // to remove redundancy.
 
-package utils_test
+package spanneraccessor_test
 
 import (
 	"context"
@@ -28,7 +28,7 @@ import (
 
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
 	"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
-	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/conversion"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
@@ -116,7 +116,7 @@ func TestCheckExistingDb(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		dbExists, err := spanneracc.CheckExistingDb(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName))
+		dbExists, err := spanneraccessor.CheckExistingDb(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName))
 		assert.Nil(t, err)
 		assert.Equal(t, tc.dbExists, dbExists)
 	}
diff --git a/webv2/helpers/helpers.go b/webv2/helpers/helpers.go
index 7495d9324a..7ad584746b 100644
--- a/webv2/helpers/helpers.go
+++ b/webv2/helpers/helpers.go
@@ -21,7 +21,7 @@ import (
 	"strings"
 
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
-	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
 )
@@ -160,7 +160,7 @@ func CheckOrCreateMetadataDb(projectId string, instanceId string) bool {
 	}
 	defer adminClient.Close()
 
-	dbExists, err := spanneracc.CheckExistingDb(ctx, uri)
+	dbExists, err := spanneraccessor.CheckExistingDb(ctx, uri)
 	if err != nil {
 		fmt.Println(err)
 		return false
diff --git a/webv2/profile/profile.go b/webv2/profile/profile.go
index e46a78ed34..634cd7a651 100644
--- a/webv2/profile/profile.go
+++ b/webv2/profile/profile.go
@@ -11,7 +11,7 @@ import (
 	"strings"
 
 	datastream "cloud.google.com/go/datastream/apiv1"
-	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/streaming"
@@ -161,7 +161,7 @@ func CreateConnectionProfile(w http.ResponseWriter, r *http.Request) {
 		} else {
 			bucketName = strings.ToLower(sessionState.Conv.Audit.MigrationRequestId)
 		}
-		err = storageacc.CreateGCSBucket(ctx, bucketName, sessionState.GCPProjectID, sessionState.Region)
+		err = storageaccessor.CreateGCSBucket(ctx, bucketName, sessionState.GCPProjectID, sessionState.Region)
 		if err != nil {
 			http.Error(w, fmt.Sprintf("Error while creating bucket: %v", err), http.StatusBadRequest)
 			return
diff --git a/webv2/session/session_service.go b/webv2/session/session_service.go
index d6a26f8685..e579dd1ec6 100644
--- a/webv2/session/session_service.go
+++ b/webv2/session/session_service.go
@@ -7,7 +7,7 @@ import (
 	"cloud.google.com/go/spanner"
 	database "cloud.google.com/go/spanner/admin/database/apiv1"
 	"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
-	spanneracc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
 	helpers "github.com/GoogleCloudPlatform/spanner-migration-tool/webv2/helpers"
 )
 
@@ -88,7 +88,7 @@ func migrateMetadataDb(projectId, instanceId string) {
 	defer adminClient.Close()
 
 	oldMetadataDbUri := getOldMetadataDbUri(projectId, instanceId)
-	oldMetadataDBExists, err := spanneracc.CheckExistingDb(ctx, oldMetadataDbUri)
+	oldMetadataDBExists, err := spanneraccessor.CheckExistingDb(ctx, oldMetadataDbUri)
 	if err != nil {
 		fmt.Printf("could not check if oldMetadataDB exists. error=%v\n", err)
 		return
diff --git a/webv2/web.go b/webv2/web.go
index 0ae24e2f33..a0cde1399d 100644
--- a/webv2/web.go
+++ b/webv2/web.go
@@ -36,7 +36,7 @@ import (
 	"time"
 
 	instance "cloud.google.com/go/spanner/admin/instance/apiv1"
-	storageacc "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/cmd"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -2487,7 +2487,7 @@ func createConfigFileForShardedBulkMigration(sessionState *session.SessionState,
 
 func writeSessionFile(ctx context.Context, sessionState *session.SessionState) error {
 
-	err := storageacc.CreateGCSBucket(ctx, sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region)
+	err := storageaccessor.CreateGCSBucket(ctx, sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region)
 	if err != nil {
 		return fmt.Errorf("error while creating bucket: %v", err)
 	}
@@ -2496,7 +2496,7 @@ func writeSessionFile(ctx context.Context, sessionState *session.SessionState) e
 	if err != nil {
 		return fmt.Errorf("can't encode session state to JSON: %v", err)
 	}
-	err = storageacc.WriteDataToGCS(ctx, "gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON))
+	err = storageaccessor.WriteDataToGCS(ctx, "gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON))
 	if err != nil {
 		return fmt.Errorf("error while writing to GCS: %v", err)
 	}

From 66940715f8b267559bf6ca131746243c720eae9d Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 10 Jan 2024 20:36:05 +0530
Subject: [PATCH 18/35] Add empty test files

---
 .../clients/spanner/admin/admin_client_test.go     | 14 ++++++++++++++
 .../clients/spanner/client/spanner_client_test.go  | 14 ++++++++++++++
 .../instanceadmin/spanner_instance_admin_test.go   | 14 ++++++++++++++
 accessors/clients/storage/storage_client_test.go   | 14 ++++++++++++++
 accessors/spanner/spanner_accessor_test.go         | 14 ++++++++++++++
 accessors/storage/storage_accessor_test.go         | 14 ++++++++++++++
 common/utils/storage_utils.go                      |  6 ++++--
 testing/accessors/spanner/spanner_accessor_test.go |  1 +
 8 files changed, 89 insertions(+), 2 deletions(-)
 create mode 100644 accessors/clients/spanner/admin/admin_client_test.go
 create mode 100644 accessors/clients/spanner/client/spanner_client_test.go
 create mode 100644 accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go
 create mode 100644 accessors/clients/storage/storage_client_test.go
 create mode 100644 accessors/spanner/spanner_accessor_test.go
 create mode 100644 accessors/storage/storage_accessor_test.go

diff --git a/accessors/clients/spanner/admin/admin_client_test.go b/accessors/clients/spanner/admin/admin_client_test.go
new file mode 100644
index 0000000000..eafb726e1a
--- /dev/null
+++ b/accessors/clients/spanner/admin/admin_client_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spanneradmin
diff --git a/accessors/clients/spanner/client/spanner_client_test.go b/accessors/clients/spanner/client/spanner_client_test.go
new file mode 100644
index 0000000000..b675039f44
--- /dev/null
+++ b/accessors/clients/spanner/client/spanner_client_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spannerclient
diff --git a/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go b/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go
new file mode 100644
index 0000000000..5bee8f5d97
--- /dev/null
+++ b/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spinstanceadmin
diff --git a/accessors/clients/storage/storage_client_test.go b/accessors/clients/storage/storage_client_test.go
new file mode 100644
index 0000000000..02bff2f3ac
--- /dev/null
+++ b/accessors/clients/storage/storage_client_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package storageclient
diff --git a/accessors/spanner/spanner_accessor_test.go b/accessors/spanner/spanner_accessor_test.go
new file mode 100644
index 0000000000..5f0d2f60b9
--- /dev/null
+++ b/accessors/spanner/spanner_accessor_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package spanneraccessor
diff --git a/accessors/storage/storage_accessor_test.go b/accessors/storage/storage_accessor_test.go
new file mode 100644
index 0000000000..1519db317b
--- /dev/null
+++ b/accessors/storage/storage_accessor_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package storageaccessor
diff --git a/common/utils/storage_utils.go b/common/utils/storage_utils.go
index 3429c6d293..3968b1731e 100644
--- a/common/utils/storage_utils.go
+++ b/common/utils/storage_utils.go
@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package utils contains common helper functions used across multiple other packages.
-// Utils should not import any Spanner migration tool packages.
+/*
+Package utils contains common helper functions used across multiple other packages.
+Utils should not import any Spanner migration tool packages.
+*/
 package utils
 
 import (
diff --git a/testing/accessors/spanner/spanner_accessor_test.go b/testing/accessors/spanner/spanner_accessor_test.go
index eb7775fcca..a4c440a0c3 100644
--- a/testing/accessors/spanner/spanner_accessor_test.go
+++ b/testing/accessors/spanner/spanner_accessor_test.go
@@ -45,6 +45,7 @@ var (
 	databaseAdmin *database.DatabaseAdminClient
 )
 
+// This test should move as a mock unit test inside accessors itself.
 func TestMain(m *testing.M) {
 	cleanup := initTests()
 	res := m.Run()

From 175321c2b1747fbde2f0bfc5b3bf821e30e105ee Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 23 Jan 2024 13:08:54 +0530
Subject: [PATCH 19/35] Increade version retention period Add log statements to
 storage accessor functions

---
 accessors/spanner/spanner_accessor.go | 15 +++++----------
 accessors/storage/storage_accessor.go | 13 ++++++++++---
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/accessors/spanner/spanner_accessor.go b/accessors/spanner/spanner_accessor.go
index 1b1594c1e6..aa03a3884e 100644
--- a/accessors/spanner/spanner_accessor.go
+++ b/accessors/spanner/spanner_accessor.go
@@ -50,12 +50,8 @@ func GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
 func CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
 	gotResponse := make(chan bool)
 	var err error
-	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
-	if err != nil {
-		return false, err
-	}
 	go func() {
-		_, err = adminClient.GetDatabase(ctx, &databasepb.GetDatabaseRequest{Name: dbURI})
+		_, err = GetDatabase(ctx, dbURI)
 		gotResponse <- true
 	}()
 	for {
@@ -122,12 +118,11 @@ func CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI stri
 		return false, err
 	}
 	stmt := spanner.Statement{
-		SQL: `SELECT * FROM information_schema.change_streams`,
+		SQL: `SELECT CHANGE_STREAM_NAME FROM information_schema.change_streams`,
 	}
 	iter := spClient.Single().Query(ctx, stmt)
 	defer iter.Stop()
-	var cs_catalog, cs_schema, cs_name string
-	var coversAll bool
+	var cs_name string
 	csExists := false
 	for {
 		row, err := iter.Next()
@@ -137,7 +132,7 @@ func CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI stri
 		if err != nil {
 			return false, fmt.Errorf("couldn't read row from change_streams table: %w", err)
 		}
-		err = row.Columns(&cs_catalog, &cs_schema, &cs_name, &coversAll)
+		err = row.Columns(&cs_name)
 		if err != nil {
 			return false, fmt.Errorf("can't scan row from change_streams table: %v", err)
 		}
@@ -189,7 +184,7 @@ func CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) err
 	op, err := spClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{
 		Database: dbURI,
 		// TODO: create change stream for only the tables present in Spanner.
-		Statements: []string{fmt.Sprintf("CREATE CHANGE STREAM %s FOR ALL OPTIONS (value_capture_type = 'NEW_ROW')", changeStreamName)},
+		Statements: []string{fmt.Sprintf("CREATE CHANGE STREAM %s FOR ALL OPTIONS (value_capture_type = 'NEW_ROW', retention_period = '7d')", changeStreamName)},
 	})
 	if err != nil {
 		return fmt.Errorf("cannot submit request create change stream request: %v", err)
diff --git a/accessors/storage/storage_accessor.go b/accessors/storage/storage_accessor.go
index f587b32627..25c0828906 100644
--- a/accessors/storage/storage_accessor.go
+++ b/accessors/storage/storage_accessor.go
@@ -75,7 +75,7 @@ func createGCSBucketUtil(ctx context.Context, bucketName, projectID, location st
 		}
 
 	} else {
-		fmt.Printf("Created new GCS bucket: %v\n", bucketName)
+		logger.Log.Info(fmt.Sprintf("Created new GCS bucket: %v\n", bucketName))
 	}
 	return nil
 }
@@ -143,10 +143,14 @@ func WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error
 	obj := bucket.Object(u.Path[1:] + fileName)
 
 	w := obj.NewWriter(ctx)
-	if _, err := fmt.Fprint(w, data); err != nil {
+	logger.Log.Info(fmt.Sprintf("Writing data to %s", filePath))
+	n, err := fmt.Fprint(w, data)
+	if err != nil {
 		fmt.Printf("Failed to write to Cloud Storage: %s", filePath)
 		return err
 	}
+	logger.Log.Info(fmt.Sprintf("Wrote %d bytes to GCS", n))
+
 	if err := w.Close(); err != nil {
 		fmt.Printf("Failed to close GCS file: %s", filePath)
 		return err
@@ -174,9 +178,12 @@ func ReadGcsFile(ctx context.Context, filePath string) (string, error) {
 	}
 	defer rc.Close()
 	buf := new(strings.Builder)
-	if _, err := io.Copy(buf, rc); err != nil {
+	logger.Log.Info(fmt.Sprintf("Reading from %s", filePath))
+	n, err := io.Copy(buf, rc)
+	if err != nil {
 		return "", err
 	}
+	logger.Log.Info(fmt.Sprintf("Read %d bytes", n))
 	return buf.String(), nil
 }
 

From d3dd9027f9e54d9bcdebbb8e43627b7138d02ed4 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 13:40:27 +0530
Subject: [PATCH 20/35] Add storage accessor interface and impl

---
 accessors/storage/storage_accessor.go      | 36 ++++++++++++++--------
 accessors/utils/dataflow/dataflow_utils.go |  4 +--
 conversion/conversion.go                   |  6 ++--
 streaming/streaming.go                     |  6 ++--
 webv2/profile/profile.go                   |  3 +-
 webv2/web.go                               |  6 ++--
 6 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/accessors/storage/storage_accessor.go b/accessors/storage/storage_accessor.go
index 25c0828906..59d29e6f9c 100644
--- a/accessors/storage/storage_accessor.go
+++ b/accessors/storage/storage_accessor.go
@@ -28,15 +28,27 @@ import (
 	"google.golang.org/api/googleapi"
 )
 
-func CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error {
-	return createGCSBucketUtil(ctx, bucketName, projectID, location, nil, 0)
+type StorageAccessor interface {
+	CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error
+	CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error
+	EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error
+	UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error
+	WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error
+	ReadGcsFile(ctx context.Context, filePath string) (string, error)
+	ReadAnyFile(ctx context.Context, filePath string) (string, error)
 }
 
-func CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
-	return createGCSBucketUtil(ctx, bucketName, projectID, location, matchesPrefix, ttl)
+type StorageAccessorImpl struct{}
+
+func (sa StorageAccessorImpl) CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error {
+	return sa.createGCSBucketUtil(ctx, bucketName, projectID, location, nil, 0)
+}
+
+func (sa StorageAccessorImpl) CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
+	return sa.createGCSBucketUtil(ctx, bucketName, projectID, location, matchesPrefix, ttl)
 }
 
-func createGCSBucketUtil(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
+func (sa StorageAccessorImpl) createGCSBucketUtil(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return err
@@ -83,7 +95,7 @@ func createGCSBucketUtil(ctx context.Context, bucketName, projectID, location st
 // Applies the bucket lifecycle with delete rule. Only accepts the Age and
 // prefix rule conditions as it is only used for the Datastream destination
 // bucket currently.
-func EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error {
+func (sa StorageAccessorImpl) EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return fmt.Errorf("could not create client while enabling lifecycle: %w", err)
@@ -120,15 +132,15 @@ func EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, mat
 }
 
 // UploadLocalFileToGCS uploads an object.
-func UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error {
+func (sa StorageAccessorImpl) UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error {
 	data, err := os.ReadFile(localFilePath)
 	if err != nil {
 		return fmt.Errorf("could not read file %s: %w", localFilePath, err)
 	}
-	return WriteDataToGCS(ctx, filePath, fileName, string(data))
+	return sa.WriteDataToGCS(ctx, filePath, fileName, string(data))
 }
 
-func WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error {
+func (sa StorageAccessorImpl) WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return fmt.Errorf("could not create client while uploading to GCS: %w", err)
@@ -158,7 +170,7 @@ func WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error
 	return nil
 }
 
-func ReadGcsFile(ctx context.Context, filePath string) (string, error) {
+func (sa StorageAccessorImpl) ReadGcsFile(ctx context.Context, filePath string) (string, error) {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return "", fmt.Errorf("could not create client: %w", err)
@@ -187,9 +199,9 @@ func ReadGcsFile(ctx context.Context, filePath string) (string, error) {
 	return buf.String(), nil
 }
 
-func ReadAnyFile(ctx context.Context, filePath string) (string, error) {
+func (sa StorageAccessorImpl) ReadAnyFile(ctx context.Context, filePath string) (string, error) {
 	if strings.HasPrefix(filePath, constants.GCS_FILE_PREFIX) {
-		return ReadGcsFile(ctx, filePath)
+		return sa.ReadGcsFile(ctx, filePath)
 	}
 	buf, err := os.ReadFile(filePath)
 	if err != nil {
diff --git a/accessors/utils/dataflow/dataflow_utils.go b/accessors/utils/dataflow/dataflow_utils.go
index 8266f720ac..5cd18d6d9a 100644
--- a/accessors/utils/dataflow/dataflow_utils.go
+++ b/accessors/utils/dataflow/dataflow_utils.go
@@ -157,8 +157,8 @@ func formatAdditionalUserLabels(labels map[string]string) string {
 	return strings.Join(res, ",")
 }
 
-func UnmarshalDataflowTuningConfig(ctx context.Context, filePath string) (dataflowaccessor.DataflowTuningConfig, error) {
-	jsonStr, err := storageaccessor.ReadAnyFile(ctx, filePath)
+func UnmarshalDataflowTuningConfig(ctx context.Context, sa storageaccessor.StorageAccessor, filePath string) (dataflowaccessor.DataflowTuningConfig, error) {
+	jsonStr, err := sa.ReadAnyFile(ctx, filePath)
 	if err != nil {
 		return dataflowaccessor.DataflowTuningConfig{}, err
 	}
diff --git a/conversion/conversion.go b/conversion/conversion.go
index c6376f0215..23610c0653 100644
--- a/conversion/conversion.go
+++ b/conversion/conversion.go
@@ -356,8 +356,9 @@ func dataFromDatabase(ctx context.Context, sourceProfile profiles.SourceProfile,
 
 			// Try to apply lifecycle rule to Datastream destination bucket.
 			gcsConfig := streamingCfg.GcsCfg
+			sa := storageaccessor.StorageAccessorImpl{}
 			if gcsConfig.TtlInDaysSet {
-				err = storageaccessor.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
+				err = sa.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
 				if err != nil {
 					logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err))
 					logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n")
@@ -477,8 +478,9 @@ func dataFromDatabaseForDataflowMigration(targetProfile profiles.TargetProfile,
 
 		// Try to apply lifecycle rule to Datastream destination bucket.
 		gcsConfig := streamingCfg.GcsCfg
+		sa := storageaccessor.StorageAccessorImpl{}
 		if gcsConfig.TtlInDaysSet {
-			err = storageaccessor.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
+			err = sa.EnableBucketLifecycleDeleteRule(ctx, gcsBucket, []string{gcsDestPrefix}, gcsConfig.TtlInDays)
 			if err != nil {
 				logger.Log.Warn(fmt.Sprintf("\nWARNING: could not update Datastream destination GCS bucket with lifecycle rule, error: %v\n", err))
 				logger.Log.Warn("Please apply the lifecycle rule manually. Continuing...\n")
diff --git a/streaming/streaming.go b/streaming/streaming.go
index 57775356a2..c25d173fdf 100644
--- a/streaming/streaming.go
+++ b/streaming/streaming.go
@@ -860,12 +860,12 @@ func StartDatastream(ctx context.Context, streamingCfg StreamingCfg, sourceProfi
 }
 
 func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, streamingCfg StreamingCfg, conv *internal.Conv) (internal.DataflowOutput, error) {
-
+	sa := storageaccessor.StorageAccessorImpl{}
 	convJSON, err := json.MarshalIndent(conv, "", " ")
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("can't encode session state to JSON: %v", err)
 	}
-	err = storageaccessor.WriteDataToGCS(ctx, streamingCfg.TmpDir, "session.json", string(convJSON))
+	err = sa.WriteDataToGCS(ctx, streamingCfg.TmpDir, "session.json", string(convJSON))
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err)
 	}
@@ -876,7 +876,7 @@ func StartDataflow(ctx context.Context, targetProfile profiles.TargetProfile, st
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("failed to compute transformation context: %s", err.Error())
 	}
-	err = storageaccessor.WriteDataToGCS(ctx, streamingCfg.TmpDir, "transformationContext.json", string(transformationContext))
+	err = sa.WriteDataToGCS(ctx, streamingCfg.TmpDir, "transformationContext.json", string(transformationContext))
 	if err != nil {
 		return internal.DataflowOutput{}, fmt.Errorf("error while writing to GCS: %v", err)
 	}
diff --git a/webv2/profile/profile.go b/webv2/profile/profile.go
index 634cd7a651..923bb5d84d 100644
--- a/webv2/profile/profile.go
+++ b/webv2/profile/profile.go
@@ -154,6 +154,7 @@ func CreateConnectionProfile(w http.ResponseWriter, r *http.Request) {
 		ValidateOnly: details.ValidateOnly,
 	}
 	var bucketName string
+	sa := storageaccessor.StorageAccessorImpl{}
 	if !details.IsSource {
 
 		if sessionState.IsSharded {
@@ -161,7 +162,7 @@ func CreateConnectionProfile(w http.ResponseWriter, r *http.Request) {
 		} else {
 			bucketName = strings.ToLower(sessionState.Conv.Audit.MigrationRequestId)
 		}
-		err = storageaccessor.CreateGCSBucket(ctx, bucketName, sessionState.GCPProjectID, sessionState.Region)
+		err = sa.CreateGCSBucket(ctx, bucketName, sessionState.GCPProjectID, sessionState.Region)
 		if err != nil {
 			http.Error(w, fmt.Sprintf("Error while creating bucket: %v", err), http.StatusBadRequest)
 			return
diff --git a/webv2/web.go b/webv2/web.go
index a0cde1399d..c45dab3514 100644
--- a/webv2/web.go
+++ b/webv2/web.go
@@ -2486,8 +2486,8 @@ func createConfigFileForShardedBulkMigration(sessionState *session.SessionState,
 }
 
 func writeSessionFile(ctx context.Context, sessionState *session.SessionState) error {
-
-	err := storageaccessor.CreateGCSBucket(ctx, sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region)
+	sa := storageaccessor.StorageAccessorImpl{}
+	err := sa.CreateGCSBucket(ctx, sessionState.Bucket, sessionState.GCPProjectID, sessionState.Region)
 	if err != nil {
 		return fmt.Errorf("error while creating bucket: %v", err)
 	}
@@ -2496,7 +2496,7 @@ func writeSessionFile(ctx context.Context, sessionState *session.SessionState) e
 	if err != nil {
 		return fmt.Errorf("can't encode session state to JSON: %v", err)
 	}
-	err = storageaccessor.WriteDataToGCS(ctx, "gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON))
+	err = sa.WriteDataToGCS(ctx, "gs://"+sessionState.Bucket+sessionState.RootPath, "session.json", string(convJSON))
 	if err != nil {
 		return fmt.Errorf("error while writing to GCS: %v", err)
 	}

From 722ac2a06c3fae18267a8ec8cd578313ba5f093a Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 14:00:44 +0530
Subject: [PATCH 21/35] Add storage client unit tests

---
 accessors/clients/storage/storage_client.go   |   6 +-
 .../clients/storage/storage_client_test.go    | 103 ++++++++++++++++++
 2 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/accessors/clients/storage/storage_client.go b/accessors/clients/storage/storage_client.go
index 569841d299..28c62b868f 100644
--- a/accessors/clients/storage/storage_client.go
+++ b/accessors/clients/storage/storage_client.go
@@ -24,11 +24,15 @@ import (
 var once sync.Once
 var gcsClient *storage.Client
 
+// This function is declared as a global variable to make it testable. The unit
+// tests edit this function, acting like a double.
+var newClient = storage.NewClient
+
 func GetOrCreateClient(ctx context.Context) (*storage.Client, error) {
 	var err error
 	if gcsClient == nil {
 		once.Do(func() {
-			gcsClient, err = storage.NewClient(ctx)
+			gcsClient, err = newClient(ctx)
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create storage client: %v", err)
diff --git a/accessors/clients/storage/storage_client_test.go b/accessors/clients/storage/storage_client_test.go
index 02bff2f3ac..73bd6b873a 100644
--- a/accessors/clients/storage/storage_client_test.go
+++ b/accessors/clients/storage/storage_client_test.go
@@ -12,3 +12,106 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package storageclient
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+
+	"cloud.google.com/go/storage"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func resetTest() {
+	gcsClient = nil
+	once = sync.Once{}
+}
+
+func TestGetOrCreateClient_Basic(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+	newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) {
+		return &storage.Client{}, nil
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) {
+		return &storage.Client{}, nil
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+	// Explicitly set the client to nil. Running GetOrCreateClient should not create a
+	// new client since sync would already be executed.
+	gcsClient = nil
+
+	newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err = GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) {
+		return &storage.Client{}, nil
+	}
+	oldC, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, oldC)
+	assert.Nil(t, err)
+
+	// Explicitly reset once. Running GetOrCreateClient should not create a
+	// new client the if condition should prevent it.
+	once = sync.Once{}
+	newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	newC, err := GetOrCreateClient(ctx)
+	assert.Equal(t, oldC, newC)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_Error(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.NotNil(t, err)
+}

From 12b7737af39459afa1c9e9c24cb27bc982667e01 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 14:10:11 +0530
Subject: [PATCH 22/35] Add spanner admin client unit tests

---
 .../clients/spanner/admin/admin_client.go     |   6 +-
 .../spanner/admin/admin_client_test.go        | 102 ++++++++++++++++++
 2 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/accessors/clients/spanner/admin/admin_client.go b/accessors/clients/spanner/admin/admin_client.go
index c0ba8aed5f..220621ee09 100644
--- a/accessors/clients/spanner/admin/admin_client.go
+++ b/accessors/clients/spanner/admin/admin_client.go
@@ -24,11 +24,15 @@ import (
 var once sync.Once
 var spannerAdminClient *database.DatabaseAdminClient
 
+// This function is declared as a global variable to make it testable. The unit
+// tests edit this function, acting like a double.
+var newDatabaseAdminClient = database.NewDatabaseAdminClient
+
 func GetOrCreateClient(ctx context.Context) (*database.DatabaseAdminClient, error) {
 	var err error
 	if spannerAdminClient == nil {
 		once.Do(func() {
-			spannerAdminClient, err = database.NewDatabaseAdminClient(ctx)
+			spannerAdminClient, err = newDatabaseAdminClient(ctx)
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create spanner admin client: %v", err)
diff --git a/accessors/clients/spanner/admin/admin_client_test.go b/accessors/clients/spanner/admin/admin_client_test.go
index eafb726e1a..7c911ee096 100644
--- a/accessors/clients/spanner/admin/admin_client_test.go
+++ b/accessors/clients/spanner/admin/admin_client_test.go
@@ -12,3 +12,105 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package spanneradmin
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+
+	database "cloud.google.com/go/spanner/admin/database/apiv1"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func resetTest() {
+	spannerAdminClient = nil
+	once = sync.Once{}
+}
+
+func TestGetOrCreateClient_Basic(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newDatabaseAdminClient
+	defer func() { newDatabaseAdminClient = oldFunc }()
+	newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) {
+		return &database.DatabaseAdminClient{}, nil
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newDatabaseAdminClient
+	defer func() { newDatabaseAdminClient = oldFunc }()
+
+	newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) {
+		return &database.DatabaseAdminClient{}, nil
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+	// Explicitly set the client to nil. Running GetOrCreateClient should not create a
+	// new client since sync would already be executed.
+	spannerAdminClient = nil
+	newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err = GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newDatabaseAdminClient
+	defer func() { newDatabaseAdminClient = oldFunc }()
+
+	newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) {
+		return &database.DatabaseAdminClient{}, nil
+	}
+	oldC, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, oldC)
+	assert.Nil(t, err)
+
+	// Explicitly reset once. Running GetOrCreateClient should not create a
+	// new client the if condition should prevent it.
+	once = sync.Once{}
+	newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	newC, err := GetOrCreateClient(ctx)
+	assert.Equal(t, oldC, newC)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_Error(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newDatabaseAdminClient
+	defer func() { newDatabaseAdminClient = oldFunc }()
+
+	newDatabaseAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*database.DatabaseAdminClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.NotNil(t, err)
+}

From 89823d6560d14d23cf49d3bafae68e64af6d40e4 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 14:13:18 +0530
Subject: [PATCH 23/35] Add spanner instance admin client unit tests

---
 .../instanceadmin/spanner_instance_admin.go   |   6 +-
 .../spanner_instance_admin_test.go            | 102 ++++++++++++++++++
 2 files changed, 107 insertions(+), 1 deletion(-)

diff --git a/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go b/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go
index 324da4ac32..8e6529bfc7 100644
--- a/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go
+++ b/accessors/clients/spanner/instanceadmin/spanner_instance_admin.go
@@ -24,11 +24,15 @@ import (
 var once sync.Once
 var instanceAdminClient *instance.InstanceAdminClient
 
+// This function is declared as a global variable to make it testable. The unit
+// tests edit this function, acting like a double.
+var newInstanceAdminClient = instance.NewInstanceAdminClient
+
 func GetOrCreateClient(ctx context.Context) (*instance.InstanceAdminClient, error) {
 	var err error
 	if instanceAdminClient == nil {
 		once.Do(func() {
-			instanceAdminClient, err = instance.NewInstanceAdminClient(ctx)
+			instanceAdminClient, err = newInstanceAdminClient(ctx)
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create spanner instance admin client: %v", err)
diff --git a/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go b/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go
index 5bee8f5d97..2792d03e82 100644
--- a/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go
+++ b/accessors/clients/spanner/instanceadmin/spanner_instance_admin_test.go
@@ -12,3 +12,105 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package spinstanceadmin
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+
+	instance "cloud.google.com/go/spanner/admin/instance/apiv1"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func resetTest() {
+	instanceAdminClient = nil
+	once = sync.Once{}
+}
+
+func TestGetOrCreateClient_Basic(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newInstanceAdminClient
+	defer func() { newInstanceAdminClient = oldFunc }()
+	newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) {
+		return &instance.InstanceAdminClient{}, nil
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newInstanceAdminClient
+	defer func() { newInstanceAdminClient = oldFunc }()
+
+	newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) {
+		return &instance.InstanceAdminClient{}, nil
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+	// Explicitly set the client to nil. Running GetOrCreateClient should not create a
+	// new client since sync would already be executed.
+	instanceAdminClient = nil
+	newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err = GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newInstanceAdminClient
+	defer func() { newInstanceAdminClient = oldFunc }()
+
+	newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) {
+		return &instance.InstanceAdminClient{}, nil
+	}
+	oldC, err := GetOrCreateClient(ctx)
+	assert.NotNil(t, oldC)
+	assert.Nil(t, err)
+
+	// Explicitly reset once. Running GetOrCreateClient should not create a
+	// new client the if condition should prevent it.
+	once = sync.Once{}
+	newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	newC, err := GetOrCreateClient(ctx)
+	assert.Equal(t, oldC, newC)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_Error(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newInstanceAdminClient
+	defer func() { newInstanceAdminClient = oldFunc }()
+
+	newInstanceAdminClient = func(ctx context.Context, opts ...option.ClientOption) (*instance.InstanceAdminClient, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err := GetOrCreateClient(ctx)
+	assert.Nil(t, c)
+	assert.NotNil(t, err)
+}

From 5bf62abfc504936e19c09b19d7d96532087a8895 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 14:19:58 +0530
Subject: [PATCH 24/35] Add spanner client unit tests

---
 .../clients/spanner/client/spanner_client.go  |   8 +-
 .../spanner/client/spanner_client_test.go     | 102 ++++++++++++++++++
 2 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/accessors/clients/spanner/client/spanner_client.go b/accessors/clients/spanner/client/spanner_client.go
index bbb3e252c1..d6edd81ece 100644
--- a/accessors/clients/spanner/client/spanner_client.go
+++ b/accessors/clients/spanner/client/spanner_client.go
@@ -24,11 +24,15 @@ import (
 var once sync.Once
 var spannerClient *sp.Client
 
+// This function is declared as a global variable to make it testable. The unit
+// tests edit this function, acting like a double.
+var newClient = sp.NewClient
+
 func GetOrCreateClient(ctx context.Context, dbURI string) (*sp.Client, error) {
 	var err error
-	if spannerClient == nil || spannerClient.DatabaseName() != dbURI {
+	if spannerClient == nil {
 		once.Do(func() {
-			spannerClient, err = sp.NewClient(ctx, dbURI)
+			spannerClient, err = newClient(ctx, dbURI)
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create spanner database client: %v", err)
diff --git a/accessors/clients/spanner/client/spanner_client_test.go b/accessors/clients/spanner/client/spanner_client_test.go
index b675039f44..66f5059591 100644
--- a/accessors/clients/spanner/client/spanner_client_test.go
+++ b/accessors/clients/spanner/client/spanner_client_test.go
@@ -12,3 +12,105 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package spannerclient
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+
+	sp "cloud.google.com/go/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func resetTest() {
+	spannerClient = nil
+	once = sync.Once{}
+}
+
+func TestGetOrCreateClient_Basic(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return &sp.Client{}, nil
+	}
+	c, err := GetOrCreateClient(ctx, "testURI")
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return &sp.Client{}, nil
+	}
+	c, err := GetOrCreateClient(ctx, "testURI")
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+	// Explicitly set the client to nil. Running GetOrCreateClient should not create a
+	// new client since sync would already be executed.
+	spannerClient = nil
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err = GetOrCreateClient(ctx, "testURI")
+	assert.Nil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return &sp.Client{}, nil
+	}
+	oldC, err := GetOrCreateClient(ctx, "testURI")
+	assert.NotNil(t, oldC)
+	assert.Nil(t, err)
+
+	// Explicitly reset once. Running GetOrCreateClient should not create a
+	// new client the if condition should prevent it.
+	once = sync.Once{}
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	newC, err := GetOrCreateClient(ctx, "testURI")
+	assert.Equal(t, oldC, newC)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_Error(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err := GetOrCreateClient(ctx, "testURI")
+	assert.Nil(t, c)
+	assert.NotNil(t, err)
+}

From 9c44518bd9087bee4e99b81ad591f2b7aa5325a5 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 16:02:25 +0530
Subject: [PATCH 25/35] Add interface and implementor for Spanner Accessor

---
 accessors/spanner/spanner_accessor.go         | 33 +++++++++++++------
 cmd/data.go                                   |  3 +-
 conversion/conversion.go                      |  3 +-
 .../spanner/spanner_accessor_test.go          |  4 +--
 webv2/helpers/helpers.go                      |  3 +-
 webv2/session/session_service.go              |  3 +-
 6 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/accessors/spanner/spanner_accessor.go b/accessors/spanner/spanner_accessor.go
index aa03a3884e..93aea4555f 100644
--- a/accessors/spanner/spanner_accessor.go
+++ b/accessors/spanner/spanner_accessor.go
@@ -29,7 +29,20 @@ import (
 	"google.golang.org/api/iterator"
 )
 
-func GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) {
+type SpannerAccessor interface {
+	GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error)
+	GetDatabaseDialect(ctx context.Context, dbURI string) (string, error)
+	CheckExistingDb(ctx context.Context, dbURI string) (bool, error)
+	CreateEmptyDatabase(ctx context.Context, dbURI string) error
+	GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error)
+	CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error)
+	ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error
+	CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error
+}
+
+type SpannerAccessorImpl struct{}
+
+func (sp SpannerAccessorImpl) GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) {
 	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
 	if err != nil {
 		return nil, err
@@ -37,8 +50,8 @@ func GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error
 	return adminClient.GetDatabase(ctx, &databasepb.GetDatabaseRequest{Name: dbURI})
 }
 
-func GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
-	result, err := GetDatabase(ctx, dbURI)
+func (sp SpannerAccessorImpl) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
+	result, err := sp.GetDatabase(ctx, dbURI)
 	if err != nil {
 		return "", fmt.Errorf("cannot connect to database: %v", err)
 	}
@@ -47,11 +60,11 @@ func GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
 
 // CheckExistingDb checks whether the database with dbURI exists or not.
 // If API call doesn't respond then user is informed after every 5 minutes on command line.
-func CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
+func (sp SpannerAccessorImpl) CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
 	gotResponse := make(chan bool)
 	var err error
 	go func() {
-		_, err = GetDatabase(ctx, dbURI)
+		_, err = sp.GetDatabase(ctx, dbURI)
 		gotResponse <- true
 	}()
 	for {
@@ -70,7 +83,7 @@ func CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
 	}
 }
 
-func CreateEmptyDatabase(ctx context.Context, dbURI string) error {
+func (sp SpannerAccessorImpl) CreateEmptyDatabase(ctx context.Context, dbURI string) error {
 	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
 	if err != nil {
 		return err
@@ -90,7 +103,7 @@ func CreateEmptyDatabase(ctx context.Context, dbURI string) error {
 	return nil
 }
 
-func GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) {
+func (sp SpannerAccessorImpl) GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) {
 	instanceClient, err := spinstanceadmin.GetOrCreateClient(ctx)
 	if err != nil {
 		return "", err
@@ -112,7 +125,7 @@ func GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string,
 	return "", fmt.Errorf("no leader found for spanner instance %s while trying fetch location", instanceURI)
 }
 
-func CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) {
+func (sp SpannerAccessorImpl) CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) {
 	spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI)
 	if err != nil {
 		return false, err
@@ -144,7 +157,7 @@ func CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI stri
 	return csExists, nil
 }
 
-func ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error {
+func (sp SpannerAccessorImpl) ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error {
 	spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI)
 	if err != nil {
 		return err
@@ -179,7 +192,7 @@ func ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI st
 	return nil
 }
 
-func CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error {
+func (sp SpannerAccessorImpl) CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error {
 	spClient, _ := spanneradmin.GetOrCreateClient(ctx)
 	op, err := spClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{
 		Database: dbURI,
diff --git a/cmd/data.go b/cmd/data.go
index c739859bf7..b31f041269 100644
--- a/cmd/data.go
+++ b/cmd/data.go
@@ -179,7 +179,8 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface
 
 // validateExistingDb validates that the existing spanner schema is in accordance with the one specified in the session file.
 func validateExistingDb(ctx context.Context, spDialect, dbURI string, adminClient *database.DatabaseAdminClient, client *sp.Client, conv *internal.Conv) error {
-	dbExists, err := spanneraccessor.CheckExistingDb(ctx, dbURI)
+	spA := spanneraccessor.SpannerAccessorImpl{}
+	dbExists, err := spA.CheckExistingDb(ctx, dbURI)
 	if err != nil {
 		err = fmt.Errorf("can't verify target database: %v", err)
 		return err
diff --git a/conversion/conversion.go b/conversion/conversion.go
index 23610c0653..bf02976781 100644
--- a/conversion/conversion.go
+++ b/conversion/conversion.go
@@ -802,7 +802,8 @@ func getSeekable(f *os.File) (*os.File, int64, error) {
 
 // VerifyDb checks whether the db exists and if it does, verifies if the schema is what we currently support.
 func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (dbExists bool, err error) {
-	dbExists, err = spanneraccessor.CheckExistingDb(ctx, dbURI)
+	spA := spanneraccessor.SpannerAccessorImpl{}
+	dbExists, err = spA.CheckExistingDb(ctx, dbURI)
 	if err != nil {
 		return dbExists, err
 	}
diff --git a/testing/accessors/spanner/spanner_accessor_test.go b/testing/accessors/spanner/spanner_accessor_test.go
index a4c440a0c3..0dd0baa07f 100644
--- a/testing/accessors/spanner/spanner_accessor_test.go
+++ b/testing/accessors/spanner/spanner_accessor_test.go
@@ -115,9 +115,9 @@ func TestCheckExistingDb(t *testing.T) {
 		{"check-db-exists", true},
 		{"check-db-does-not-exist", false},
 	}
-
+	spA := spanneraccessor.SpannerAccessorImpl{}
 	for _, tc := range testCases {
-		dbExists, err := spanneraccessor.CheckExistingDb(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName))
+		dbExists, err := spA.CheckExistingDb(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, tc.dbName))
 		assert.Nil(t, err)
 		assert.Equal(t, tc.dbExists, dbExists)
 	}
diff --git a/webv2/helpers/helpers.go b/webv2/helpers/helpers.go
index 7ad584746b..0511070cb3 100644
--- a/webv2/helpers/helpers.go
+++ b/webv2/helpers/helpers.go
@@ -160,7 +160,8 @@ func CheckOrCreateMetadataDb(projectId string, instanceId string) bool {
 	}
 	defer adminClient.Close()
 
-	dbExists, err := spanneraccessor.CheckExistingDb(ctx, uri)
+	spA := spanneraccessor.SpannerAccessorImpl{}
+	dbExists, err := spA.CheckExistingDb(ctx, uri)
 	if err != nil {
 		fmt.Println(err)
 		return false
diff --git a/webv2/session/session_service.go b/webv2/session/session_service.go
index e579dd1ec6..705e2284dc 100644
--- a/webv2/session/session_service.go
+++ b/webv2/session/session_service.go
@@ -87,8 +87,9 @@ func migrateMetadataDb(projectId, instanceId string) {
 	}
 	defer adminClient.Close()
 
+	spA := spanneraccessor.SpannerAccessorImpl{}
 	oldMetadataDbUri := getOldMetadataDbUri(projectId, instanceId)
-	oldMetadataDBExists, err := spanneraccessor.CheckExistingDb(ctx, oldMetadataDbUri)
+	oldMetadataDBExists, err := spA.CheckExistingDb(ctx, oldMetadataDbUri)
 	if err != nil {
 		fmt.Printf("could not check if oldMetadataDB exists. error=%v\n", err)
 		return

From 9267a1386b3d1e58413eee8384bf43dcac2d3702 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 16:02:44 +0530
Subject: [PATCH 26/35] Add unit test for storage utils

---
 common/utils/storage_utils_test.go | 87 ++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 common/utils/storage_utils_test.go

diff --git a/common/utils/storage_utils_test.go b/common/utils/storage_utils_test.go
new file mode 100644
index 0000000000..c9ea7f75a5
--- /dev/null
+++ b/common/utils/storage_utils_test.go
@@ -0,0 +1,87 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package utils
+
+import (
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func TestParseGCSFilePath(t *testing.T) {
+	testCases := []struct {
+		name        string
+		filePath    string
+		expectError bool
+		want        *url.URL
+	}{
+		{
+			name:        "Basic",
+			filePath:    "gs://test-bucket/path/to/folder/",
+			expectError: false,
+			want: &url.URL{
+				Scheme: "gs",
+				Host:   "test-bucket",
+				Path:   "/path/to/folder/",
+			},
+		},
+		{
+			name:        "Append Slash",
+			filePath:    "gs://test-bucket/path/to/folder",
+			expectError: false,
+			want: &url.URL{
+				Scheme: "gs",
+				Host:   "test-bucket",
+				Path:   "/path/to/folder/",
+			},
+		},
+		{
+			name:        "Empty File path",
+			filePath:    "",
+			expectError: true,
+			want:        nil,
+		},
+		{
+			name:        "Wrong Scheme",
+			filePath:    "ab://testpath",
+			expectError: true,
+			want:        nil,
+		},
+		{
+			name:        "Malformed Path",
+			filePath:    "://path",
+			expectError: true,
+			want:        nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		got, err := ParseGCSFilePath(tc.filePath)
+		assert.Equal(t, tc.expectError, err != nil)
+		assert.Equal(t, tc.want, got)
+	}
+}

From 69534f19172c92d1946a0843d3288f7b3ce67ae2 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Wed, 24 Jan 2024 16:57:03 +0530
Subject: [PATCH 27/35] Add unit test for dataflow
 utils:UnmarshalDataflowConfig

---
 accessors/spanner/spanner_accessor.go         |  16 +--
 accessors/storage/storage_accessor.go         |  16 +--
 accessors/utils/dataflow/dataflow_utils.go    |   3 +
 .../utils/dataflow/dataflow_utils_test.go     | 111 ++++++++++++++++++
 4 files changed, 130 insertions(+), 16 deletions(-)

diff --git a/accessors/spanner/spanner_accessor.go b/accessors/spanner/spanner_accessor.go
index 93aea4555f..a5972ae036 100644
--- a/accessors/spanner/spanner_accessor.go
+++ b/accessors/spanner/spanner_accessor.go
@@ -42,7 +42,7 @@ type SpannerAccessor interface {
 
 type SpannerAccessorImpl struct{}
 
-func (sp SpannerAccessorImpl) GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) {
+func (sp *SpannerAccessorImpl) GetDatabase(ctx context.Context, dbURI string) (*databasepb.Database, error) {
 	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
 	if err != nil {
 		return nil, err
@@ -50,7 +50,7 @@ func (sp SpannerAccessorImpl) GetDatabase(ctx context.Context, dbURI string) (*d
 	return adminClient.GetDatabase(ctx, &databasepb.GetDatabaseRequest{Name: dbURI})
 }
 
-func (sp SpannerAccessorImpl) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
+func (sp *SpannerAccessorImpl) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
 	result, err := sp.GetDatabase(ctx, dbURI)
 	if err != nil {
 		return "", fmt.Errorf("cannot connect to database: %v", err)
@@ -60,7 +60,7 @@ func (sp SpannerAccessorImpl) GetDatabaseDialect(ctx context.Context, dbURI stri
 
 // CheckExistingDb checks whether the database with dbURI exists or not.
 // If API call doesn't respond then user is informed after every 5 minutes on command line.
-func (sp SpannerAccessorImpl) CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
+func (sp *SpannerAccessorImpl) CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
 	gotResponse := make(chan bool)
 	var err error
 	go func() {
@@ -83,7 +83,7 @@ func (sp SpannerAccessorImpl) CheckExistingDb(ctx context.Context, dbURI string)
 	}
 }
 
-func (sp SpannerAccessorImpl) CreateEmptyDatabase(ctx context.Context, dbURI string) error {
+func (sp *SpannerAccessorImpl) CreateEmptyDatabase(ctx context.Context, dbURI string) error {
 	adminClient, err := spanneradmin.GetOrCreateClient(ctx)
 	if err != nil {
 		return err
@@ -103,7 +103,7 @@ func (sp SpannerAccessorImpl) CreateEmptyDatabase(ctx context.Context, dbURI str
 	return nil
 }
 
-func (sp SpannerAccessorImpl) GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) {
+func (sp *SpannerAccessorImpl) GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) {
 	instanceClient, err := spinstanceadmin.GetOrCreateClient(ctx)
 	if err != nil {
 		return "", err
@@ -125,7 +125,7 @@ func (sp SpannerAccessorImpl) GetSpannerLeaderLocation(ctx context.Context, inst
 	return "", fmt.Errorf("no leader found for spanner instance %s while trying fetch location", instanceURI)
 }
 
-func (sp SpannerAccessorImpl) CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) {
+func (sp *SpannerAccessorImpl) CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) {
 	spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI)
 	if err != nil {
 		return false, err
@@ -157,7 +157,7 @@ func (sp SpannerAccessorImpl) CheckIfChangeStreamExists(ctx context.Context, cha
 	return csExists, nil
 }
 
-func (sp SpannerAccessorImpl) ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error {
+func (sp *SpannerAccessorImpl) ValidateChangeStreamOptions(ctx context.Context, changeStreamName, dbURI string) error {
 	spClient, err := spannerclient.GetOrCreateClient(ctx, dbURI)
 	if err != nil {
 		return err
@@ -192,7 +192,7 @@ func (sp SpannerAccessorImpl) ValidateChangeStreamOptions(ctx context.Context, c
 	return nil
 }
 
-func (sp SpannerAccessorImpl) CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error {
+func (sp *SpannerAccessorImpl) CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error {
 	spClient, _ := spanneradmin.GetOrCreateClient(ctx)
 	op, err := spClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{
 		Database: dbURI,
diff --git a/accessors/storage/storage_accessor.go b/accessors/storage/storage_accessor.go
index 59d29e6f9c..dc0d404bb3 100644
--- a/accessors/storage/storage_accessor.go
+++ b/accessors/storage/storage_accessor.go
@@ -40,15 +40,15 @@ type StorageAccessor interface {
 
 type StorageAccessorImpl struct{}
 
-func (sa StorageAccessorImpl) CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error {
+func (sa *StorageAccessorImpl) CreateGCSBucket(ctx context.Context, bucketName, projectID, location string) error {
 	return sa.createGCSBucketUtil(ctx, bucketName, projectID, location, nil, 0)
 }
 
-func (sa StorageAccessorImpl) CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
+func (sa *StorageAccessorImpl) CreateGCSBucketWithLifecycle(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
 	return sa.createGCSBucketUtil(ctx, bucketName, projectID, location, matchesPrefix, ttl)
 }
 
-func (sa StorageAccessorImpl) createGCSBucketUtil(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
+func (sa *StorageAccessorImpl) createGCSBucketUtil(ctx context.Context, bucketName, projectID, location string, matchesPrefix []string, ttl int64) error {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return err
@@ -95,7 +95,7 @@ func (sa StorageAccessorImpl) createGCSBucketUtil(ctx context.Context, bucketNam
 // Applies the bucket lifecycle with delete rule. Only accepts the Age and
 // prefix rule conditions as it is only used for the Datastream destination
 // bucket currently.
-func (sa StorageAccessorImpl) EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error {
+func (sa *StorageAccessorImpl) EnableBucketLifecycleDeleteRule(ctx context.Context, bucketName string, matchesPrefix []string, ttl int64) error {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return fmt.Errorf("could not create client while enabling lifecycle: %w", err)
@@ -132,7 +132,7 @@ func (sa StorageAccessorImpl) EnableBucketLifecycleDeleteRule(ctx context.Contex
 }
 
 // UploadLocalFileToGCS uploads an object.
-func (sa StorageAccessorImpl) UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error {
+func (sa *StorageAccessorImpl) UploadLocalFileToGCS(ctx context.Context, filePath, fileName, localFilePath string) error {
 	data, err := os.ReadFile(localFilePath)
 	if err != nil {
 		return fmt.Errorf("could not read file %s: %w", localFilePath, err)
@@ -140,7 +140,7 @@ func (sa StorageAccessorImpl) UploadLocalFileToGCS(ctx context.Context, filePath
 	return sa.WriteDataToGCS(ctx, filePath, fileName, string(data))
 }
 
-func (sa StorageAccessorImpl) WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error {
+func (sa *StorageAccessorImpl) WriteDataToGCS(ctx context.Context, filePath, fileName, data string) error {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return fmt.Errorf("could not create client while uploading to GCS: %w", err)
@@ -170,7 +170,7 @@ func (sa StorageAccessorImpl) WriteDataToGCS(ctx context.Context, filePath, file
 	return nil
 }
 
-func (sa StorageAccessorImpl) ReadGcsFile(ctx context.Context, filePath string) (string, error) {
+func (sa *StorageAccessorImpl) ReadGcsFile(ctx context.Context, filePath string) (string, error) {
 	client, err := storageclient.GetOrCreateClient(ctx)
 	if err != nil {
 		return "", fmt.Errorf("could not create client: %w", err)
@@ -199,7 +199,7 @@ func (sa StorageAccessorImpl) ReadGcsFile(ctx context.Context, filePath string)
 	return buf.String(), nil
 }
 
-func (sa StorageAccessorImpl) ReadAnyFile(ctx context.Context, filePath string) (string, error) {
+func (sa *StorageAccessorImpl) ReadAnyFile(ctx context.Context, filePath string) (string, error) {
 	if strings.HasPrefix(filePath, constants.GCS_FILE_PREFIX) {
 		return sa.ReadGcsFile(ctx, filePath)
 	}
diff --git a/accessors/utils/dataflow/dataflow_utils.go b/accessors/utils/dataflow/dataflow_utils.go
index 5cd18d6d9a..2a6a9f69b9 100644
--- a/accessors/utils/dataflow/dataflow_utils.go
+++ b/accessors/utils/dataflow/dataflow_utils.go
@@ -11,6 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+// This is a package is kept with accessors because some functions import other accessors.
+// The common/utils package should not import any SMT dependency.
 package dataflowutils
 
 import (
diff --git a/accessors/utils/dataflow/dataflow_utils_test.go b/accessors/utils/dataflow/dataflow_utils_test.go
index a6bc771c9c..2d1e33ceff 100644
--- a/accessors/utils/dataflow/dataflow_utils_test.go
+++ b/accessors/utils/dataflow/dataflow_utils_test.go
@@ -14,11 +14,14 @@
 package dataflowutils
 
 import (
+	"context"
+	"fmt"
 	"os"
 	"testing"
 
 	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
 	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
@@ -220,3 +223,111 @@ func EquateLaunchFlexTemplateRequest(df1 *dataflowpb.LaunchFlexTemplateRequest,
 		cmp.Equal(lp1.Environment.AdditionalExperiments, lp2.Environment.AdditionalExperiments) &&
 		lp1.GetContainerSpecGcsPath() == lp2.GetContainerSpecGcsPath())
 }
+
+type StorageAccessorMock struct {
+	storageaccessor.StorageAccessorImpl
+}
+
+var readAnyFileMock func(ctx context.Context, filePath string) (string, error)
+
+func (sam StorageAccessorMock) ReadAnyFile(ctx context.Context, filePath string) (string, error) {
+	return readAnyFileMock(ctx, filePath)
+}
+
+func TestUnmarshalDataflowTuningConfig(t *testing.T) {
+	testCases := []struct {
+		name            string
+		readAnyFileMock func(ctx context.Context, filePath string) (string, error)
+		expectError     bool
+		want            dataflowaccessor.DataflowTuningConfig
+	}{
+		{
+			name: "Basic",
+			readAnyFileMock: func(ctx context.Context, filePath string) (string, error) {
+				return `{
+					"projectId": "test-project",
+					"jobName": "test-job-name",
+					"location": "us-central1",
+					"network": "test-network",
+					"subnetwork": "test-subnetwork",
+					"hostProjectId": "test-host-project",
+					"maxWorkers": 3,
+					"numWorkers": 2,
+					"serviceAccountEmail": "abc@xyz.com",
+					"machineType": "n1-standard-8",
+					"additionalUserLabels": {"my": "label"},
+					"kmsKeyName": "test-key",
+					"gcsTemplatePath": "gs://path",
+					"additionalExperiments": ["xyz","123"],
+					"enableStreamingEngine": true
+				}`, nil
+			},
+			expectError: false,
+			want: dataflowaccessor.DataflowTuningConfig{
+				ProjectId:             "test-project",
+				JobName:               "test-job-name",
+				Location:              "us-central1",
+				Network:               "test-network",
+				Subnetwork:            "test-subnetwork",
+				VpcHostProjectId:      "test-host-project",
+				MaxWorkers:            3,
+				NumWorkers:            2,
+				ServiceAccountEmail:   "abc@xyz.com",
+				MachineType:           "n1-standard-8",
+				AdditionalUserLabels:  map[string]string{"my": "label"},
+				KmsKeyName:            "test-key",
+				GcsTemplatePath:       "gs://path",
+				AdditionalExperiments: []string{"xyz", "123"},
+				EnableStreamingEngine: true,
+			},
+		},
+		{
+			name: "Defaults",
+			readAnyFileMock: func(ctx context.Context, filePath string) (string, error) {
+				return `{}`, nil
+			},
+			expectError: false,
+			want: dataflowaccessor.DataflowTuningConfig{
+				ProjectId:             "",
+				JobName:               "",
+				Location:              "",
+				Network:               "",
+				Subnetwork:            "",
+				VpcHostProjectId:      "",
+				MaxWorkers:            0,
+				NumWorkers:            0,
+				ServiceAccountEmail:   "",
+				MachineType:           "",
+				AdditionalUserLabels:  nil,
+				KmsKeyName:            "",
+				GcsTemplatePath:       "",
+				AdditionalExperiments: nil,
+				EnableStreamingEngine: false,
+			},
+		},
+		{
+			name: "ReadAnyFile throws error",
+			readAnyFileMock: func(ctx context.Context, filePath string) (string, error) {
+				return "", fmt.Errorf("test error")
+			},
+			expectError: true,
+			want:        dataflowaccessor.DataflowTuningConfig{},
+		},
+		{
+			name: "Json unmarshall throws error",
+			readAnyFileMock: func(ctx context.Context, filePath string) (string, error) {
+				return "{\"abc\"", nil
+			},
+			expectError: true,
+			want:        dataflowaccessor.DataflowTuningConfig{},
+		},
+	}
+	ctx := context.Background()
+	saMock := StorageAccessorMock{}
+	for _, tc := range testCases {
+		readAnyFileMock = tc.readAnyFileMock
+		got, err := UnmarshalDataflowTuningConfig(ctx, &saMock, "unused/path/due/to/mock")
+		assert.Equal(t, tc.expectError, err != nil)
+		assert.Equal(t, tc.want, got)
+	}
+}

From 9f2a931848aa87a71e02389926072635dd8c3cdd Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:19:27 +0530
Subject: [PATCH 28/35] Add dao

---
 dao/dao.go             | 320 +++++++++++++++++++++++++++++++++++++++++
 dao/dao_client.go      |  48 +++++++
 dao/dao_client_test.go | 116 +++++++++++++++
 3 files changed, 484 insertions(+)
 create mode 100644 dao/dao.go
 create mode 100644 dao/dao_client.go
 create mode 100644 dao/dao_client_test.go

diff --git a/dao/dao.go b/dao/dao.go
new file mode 100644
index 0000000000..c515e39539
--- /dev/null
+++ b/dao/dao.go
@@ -0,0 +1,320 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"cloud.google.com/go/spanner"
+	"google.golang.org/api/iterator"
+)
+
+type StateData struct {
+	State string `json:"state"`
+}
+
+type DAO interface {
+	InsertSMTJobEntry(ctx context.Context, jobId, jobName, jobType, dialect, dbName string, jobData spanner.NullJSON) error
+	UpdateSMTJobState(ctx context.Context, jobId, state string) error
+	InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId, resourceName, resourceType string, resourceData spanner.NullJSON) error
+	UpdateSMTResourceState(ctx context.Context, resourceId, state string) error
+	UpdateSMTResourceExternalId(ctx context.Context, resourceId, externalId string) error
+}
+
+type DAOImpl struct{}
+
+// Insert a job entry into the SMT_JOB table.
+func (dao *DAOImpl) InsertSMTJobEntry(ctx context.Context, jobId, jobName, jobType, dialect, dbName string, jobData spanner.NullJSON) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `INSERT INTO SMT_JOB 
+			(JobId, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt, UpdatedAt)
+			VALUES(
+			 @jobId, @jobName, @jobType, @jobStateData, @jobData, @dialect, @dbName, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP()
+			);`,
+			Params: map[string]interface{}{
+				"jobId":        jobId,
+				"jobName":      jobName,
+				"jobType":      jobType,
+				"jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}},
+				"jobData":      jobData,
+				"dialect":      dialect,
+				"dbName":       dbName,
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		// Update job history table within the same txn.
+		_, err = updateJobHistoryWithinTxn(ctx, txn, jobId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("could not insert SMT job entry: %v", err)
+	}
+	return nil
+}
+
+// Update the state of the SMT job.
+func (dao *DAOImpl) UpdateSMTJobState(ctx context.Context, jobId, state string) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `UPDATE SMT_JOB SET JobStateData = @jobStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
+			WHERE JobId = @jobId;`,
+			Params: map[string]interface{}{
+				"jobId":        jobId,
+				"jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}},
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		_, err = updateJobHistoryWithinTxn(ctx, txn, jobId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error updating smt job state: %v", err)
+	}
+	return nil
+}
+
+// Insert an entry into the SMT_RESOURCE table.
+func (dao *DAOImpl) InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId, resourceName, resourceType string, resourceData spanner.NullJSON) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		resourceStmt := spanner.Statement{
+			SQL: `INSERT INTO SMT_RESOURCE
+			(ResourceId, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt, UpdatedAt)
+			VALUES(
+			 @resourceId, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP()
+			);`,
+			Params: map[string]interface{}{
+				"resourceId":        resourceId,
+				"jobId":             jobId,
+				"externalId":        externalId,
+				"resourceName":      resourceName,
+				"resourceType":      resourceType,
+				"resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}},
+				"resourceData":      resourceData,
+			},
+		}
+		_, err := txn.Update(ctx, resourceStmt)
+		if err != nil {
+			return err
+		}
+		// Update the resource history table in the same transaction.
+		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error inserting smt resource entry: %v", err)
+	}
+	return nil
+}
+
+// Update the state of the SMT resource.
+func (dao *DAOImpl) UpdateSMTResourceState(ctx context.Context, resourceId, state string) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `UPDATE SMT_RESOURCE SET ResourceStateData = @resourceStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
+			WHERE ResourceId = @resourceId;`,
+			Params: map[string]interface{}{
+				"resourceId":        resourceId,
+				"resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}},
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error updating smt resource state: %v", err)
+	}
+	return nil
+}
+
+// Update the external of the SMT resource.
+func (dao *DAOImpl) UpdateSMTResourceExternalId(ctx context.Context, resourceId, externalId string) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `UPDATE SMT_RESOURCE SET ExternalId = @externalId, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
+			WHERE ResourceId = @resourceId;`,
+			Params: map[string]interface{}{
+				"resourceId": resourceId,
+				"externalId": externalId,
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error updating smt resource external id: %v", err)
+	}
+	return nil
+}
+
+func updateJobHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) {
+	version, err := getLatestJobVersionWithinTxn(ctx, txn, jobId)
+	if err != nil {
+		return 0, fmt.Errorf("error fetching latest job version: %v", err)
+	}
+	// Fetch the newly updated row from SMT_JOB table.
+	stmt := spanner.Statement{SQL: `
+		SELECT 
+			JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName 
+		FROM SMT_JOB WHERE JobId = @jobId;`,
+		Params: map[string]interface{}{"jobId": jobId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	var jobName, jobType, dialect, spannerDatabaseName spanner.NullString
+	var jobStateData, jobData spanner.NullJSON
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&jobName, &jobType, &jobStateData, &jobData, &dialect, &spannerDatabaseName); err != nil {
+		return 0, fmt.Errorf("error reading smt job row: %v", err)
+	}
+
+	// Insert entry to SMT_JOB_HISTORY table.
+	jobStmt := spanner.Statement{
+		SQL: `INSERT INTO SMT_JOB_HISTORY 
+		(JobId, Version, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt)
+		VALUES(
+		 @jobId, @version, @jobName, @jobType, @jobStateData, @jobData, @dialect, @spannerDatabaseName, PENDING_COMMIT_TIMESTAMP()
+		);`,
+		Params: map[string]interface{}{
+			"jobId":               jobId,
+			"version":             version + 1,
+			"jobName":             jobName,
+			"jobType":             jobType,
+			"jobStateData":        jobStateData,
+			"jobData":             jobData,
+			"dialect":             dialect,
+			"spannerDatabaseName": spannerDatabaseName,
+		},
+	}
+	return txn.Update(ctx, jobStmt)
+}
+
+func getLatestJobVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) {
+	// Fetch latest version for the job from history table.
+	stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_JOB_HISTORY WHERE JobId = @jobId;`,
+		Params: map[string]interface{}{"jobId": jobId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	version := spanner.NullInt64{}
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&version); err != nil {
+		return 0, err
+	}
+	if version.Valid {
+		return version.Int64, nil
+	}
+	return 0, nil
+}
+
+func updateResourceHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) {
+	version, err := getLatestResourceVersionWithinTxn(ctx, txn, resourceId)
+	if err != nil {
+		return 0, fmt.Errorf("error fetching latest resource version: %v", err)
+	}
+	// Fetch the newly updated row from SMT_RESOURCE table.
+	stmt := spanner.Statement{SQL: `
+		SELECT 
+			JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData 
+		FROM SMT_RESOURCE WHERE ResourceId = @resourceId;`,
+		Params: map[string]interface{}{"resourceId": resourceId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	var jobId, externalId, resourceName, resourceType spanner.NullString
+	var resourceStateData, resourceData spanner.NullJSON
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&jobId, &externalId, &resourceName, &resourceType, &resourceStateData, &resourceData); err != nil {
+		return 0, fmt.Errorf("error reading smt resource row: %v", err)
+	}
+	// Create new entry into the SMT_RESOURCE_HISTORY table.
+	jobStmt := spanner.Statement{
+		SQL: `INSERT INTO SMT_RESOURCE_HISTORY 
+		(ResourceId, Version, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt)
+		VALUES(
+		 @resourceId, @version, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP()
+		);`,
+		Params: map[string]interface{}{
+			"resourceId":        resourceId,
+			"version":           version + 1,
+			"jobId":             jobId,
+			"externalId":        externalId,
+			"resourceName":      resourceName,
+			"resourceType":      resourceType,
+			"resourceStateData": resourceStateData,
+			"resourceData":      resourceData,
+		},
+	}
+	return txn.Update(ctx, jobStmt)
+}
+
+func getLatestResourceVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) {
+	// Fetch latest version for the resource from history table.
+	stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_RESOURCE_HISTORY WHERE ResourceId = @resourceId;`,
+		Params: map[string]interface{}{"resourceId": resourceId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	version := spanner.NullInt64{}
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&version); err != nil {
+		return 0, err
+	}
+	if version.Valid {
+		return version.Int64, nil
+	}
+	return 0, nil
+}
diff --git a/dao/dao_client.go b/dao/dao_client.go
new file mode 100644
index 0000000000..46964888de
--- /dev/null
+++ b/dao/dao_client.go
@@ -0,0 +1,48 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dao
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	sp "cloud.google.com/go/spanner"
+)
+
+var once sync.Once
+var spClient *sp.Client
+
+// This function is declared as a global variable to make it testable. The unit
+// tests edit this function, acting like a double.
+var newClient = sp.NewClient
+
+func GetOrCreateClient(ctx context.Context, dbURI string) (*sp.Client, error) {
+	var err error
+	if spClient == nil {
+		once.Do(func() {
+			spClient, err = newClient(ctx, dbURI)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create spanner database client: %v", err)
+		}
+		return spClient, nil
+	}
+	return spClient, nil
+}
+
+// The DAO client must be initiated via GetOrCreateClient() once before using GetClient().
+func GetClient() *sp.Client {
+	return spClient
+}
diff --git a/dao/dao_client_test.go b/dao/dao_client_test.go
new file mode 100644
index 0000000000..05c0352a9a
--- /dev/null
+++ b/dao/dao_client_test.go
@@ -0,0 +1,116 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dao
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+
+	sp "cloud.google.com/go/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+func resetTest() {
+	spClient = nil
+	once = sync.Once{}
+}
+
+func TestGetOrCreateClient_Basic(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return &sp.Client{}, nil
+	}
+	c, err := GetOrCreateClient(ctx, "testURI")
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaSync(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return &sp.Client{}, nil
+	}
+	c, err := GetOrCreateClient(ctx, "testURI")
+	assert.NotNil(t, c)
+	assert.Nil(t, err)
+	// Explicitly set the client to nil. Running GetOrCreateClient should not create a
+	// new client since sync would already be executed.
+	spClient = nil
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err = GetOrCreateClient(ctx, "testURI")
+	assert.Nil(t, c)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_OnlyOnceViaIf(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return &sp.Client{}, nil
+	}
+	oldC, err := GetOrCreateClient(ctx, "testURI")
+	assert.NotNil(t, oldC)
+	assert.Nil(t, err)
+
+	// Explicitly reset once. Running GetOrCreateClient should not create a
+	// new client the if condition should prevent it.
+	once = sync.Once{}
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	newC, err := GetOrCreateClient(ctx, "testURI")
+	assert.Equal(t, oldC, newC)
+	assert.Nil(t, err)
+}
+
+func TestGetOrCreateClient_Error(t *testing.T) {
+	resetTest()
+	ctx := context.Background()
+	oldFunc := newClient
+	defer func() { newClient = oldFunc }()
+
+	newClient = func(ctx context.Context, database string, opts ...option.ClientOption) (*sp.Client, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	c, err := GetOrCreateClient(ctx, "testURI")
+	assert.Nil(t, c)
+	assert.NotNil(t, err)
+}

From 58ff46700662499ccda056588ccc589c7b253b89 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:19:27 +0530
Subject: [PATCH 29/35] Add dao

---
 dao/job_metadata.go      | 155 ++++++++++++++++++++++++++++++++++
 dao/resource_metadata.go | 177 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 332 insertions(+)
 create mode 100644 dao/job_metadata.go
 create mode 100644 dao/resource_metadata.go

diff --git a/dao/job_metadata.go b/dao/job_metadata.go
new file mode 100644
index 0000000000..6ea0c6e48e
--- /dev/null
+++ b/dao/job_metadata.go
@@ -0,0 +1,155 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"cloud.google.com/go/spanner"
+	"google.golang.org/api/iterator"
+)
+
+type StateData struct {
+	State string `json:"state"`
+}
+
+// Insert a job entry into the SMT_JOB table.
+func InsertSMTJobEntry(ctx context.Context, jobId, jobName, jobType, dialect, dbName string, jobData spanner.NullJSON) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `INSERT INTO SMT_JOB 
+			(JobId, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt, UpdatedAt)
+			VALUES(
+			 @jobId, @jobName, @jobType, @jobStateData, @jobData, @dialect, @dbName, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP()
+			);`,
+			Params: map[string]interface{}{
+				"jobId":        jobId,
+				"jobName":      jobName,
+				"jobType":      jobType,
+				"jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}},
+				"jobData":      jobData,
+				"dialect":      dialect,
+				"dbName":       dbName,
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		// Update job history table within the same txn.
+		_, err = updateJobHistoryWithinTxn(ctx, txn, jobId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("could not insert SMT job entry: %v", err)
+	}
+	return nil
+}
+
+func updateJobHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) {
+	version, err := getLatestJobVersionWithinTxn(ctx, txn, jobId)
+	if err != nil {
+		return 0, fmt.Errorf("error fetching latest job version: %v", err)
+	}
+	// Fetch the newly updated row from SMT_JOB table.
+	stmt := spanner.Statement{SQL: `
+		SELECT 
+			JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName 
+		FROM SMT_JOB WHERE JobId = @jobId;`,
+		Params: map[string]interface{}{"jobId": jobId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	var jobName, jobType, dialect, spannerDatabaseName spanner.NullString
+	var jobStateData, jobData spanner.NullJSON
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&jobName, &jobType, &jobStateData, &jobData, &dialect, &spannerDatabaseName); err != nil {
+		return 0, fmt.Errorf("error reading smt job row: %v", err)
+	}
+
+	// Insert entry to SMT_JOB_HISTORY table.
+	jobStmt := spanner.Statement{
+		SQL: `INSERT INTO SMT_JOB_HISTORY 
+		(JobId, Version, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt)
+		VALUES(
+		 @jobId, @version, @jobName, @jobType, @jobStateData, @jobData, @dialect, @spannerDatabaseName, PENDING_COMMIT_TIMESTAMP()
+		);`,
+		Params: map[string]interface{}{
+			"jobId":               jobId,
+			"version":             version + 1,
+			"jobName":             jobName,
+			"jobType":             jobType,
+			"jobStateData":        jobStateData,
+			"jobData":             jobData,
+			"dialect":             dialect,
+			"spannerDatabaseName": spannerDatabaseName,
+		},
+	}
+	return txn.Update(ctx, jobStmt)
+}
+
+func getLatestJobVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) {
+	// Fetch latest version for the job from history table.
+	stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_JOB_HISTORY WHERE JobId = @jobId;`,
+		Params: map[string]interface{}{"jobId": jobId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	version := spanner.NullInt64{}
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&version); err != nil {
+		return 0, err
+	}
+	if version.Valid {
+		return version.Int64, nil
+	}
+	return 0, nil
+}
+
+// Update the state of the SMT job.
+func UpdateSMTJobState(ctx context.Context, jobId, state string) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `UPDATE SMT_JOB SET JobStateData = @jobStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
+			WHERE JobId = @jobId;`,
+			Params: map[string]interface{}{
+				"jobId":        jobId,
+				"jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}},
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		_, err = updateJobHistoryWithinTxn(ctx, txn, jobId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error updating smt job state: %v", err)
+	}
+	return nil
+}
diff --git a/dao/resource_metadata.go b/dao/resource_metadata.go
new file mode 100644
index 0000000000..1c72b298fd
--- /dev/null
+++ b/dao/resource_metadata.go
@@ -0,0 +1,177 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"cloud.google.com/go/spanner"
+	"google.golang.org/api/iterator"
+)
+
+// Insert an entry into the SMT_RESOURCE table.
+func InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId, resourceName, resourceType string, resourceData spanner.NullJSON) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		resourceStmt := spanner.Statement{
+			SQL: `INSERT INTO SMT_RESOURCE
+			(ResourceId, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt, UpdatedAt)
+			VALUES(
+			 @resourceId, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP()
+			);`,
+			Params: map[string]interface{}{
+				"resourceId":        resourceId,
+				"jobId":             jobId,
+				"externalId":        externalId,
+				"resourceName":      resourceName,
+				"resourceType":      resourceType,
+				"resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}},
+				"resourceData":      resourceData,
+			},
+		}
+		_, err := txn.Update(ctx, resourceStmt)
+		if err != nil {
+			return err
+		}
+		// Update the resoruce history table in the same transaction.
+		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error inserting smt resource entry: %v", err)
+	}
+	return nil
+}
+
+func updateResourceHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) {
+	version, err := getLatestResourceVersionWithinTxn(ctx, txn, resourceId)
+	if err != nil {
+		return 0, fmt.Errorf("error fetching latest resource version: %v", err)
+	}
+	// Fetch the newly updated row from SMT_RESOURCE table.
+	stmt := spanner.Statement{SQL: `
+		SELECT 
+			JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData 
+		FROM SMT_RESOURCE WHERE ResourceId = @resourceId;`,
+		Params: map[string]interface{}{"resourceId": resourceId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	var jobId, externalId, resourceName, resourceType spanner.NullString
+	var resourceStateData, resourceData spanner.NullJSON
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&jobId, &externalId, &resourceName, &resourceType, &resourceStateData, &resourceData); err != nil {
+		return 0, fmt.Errorf("error reading smt resource row: %v", err)
+	}
+	// Create new entry into the SMT_RESOURCE_HISTORY table.
+	jobStmt := spanner.Statement{
+		SQL: `INSERT INTO SMT_RESOURCE_HISTORY 
+		(ResourceId, Version, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt)
+		VALUES(
+		 @resourceId, @version, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP()
+		);`,
+		Params: map[string]interface{}{
+			"resourceId":        resourceId,
+			"version":           version + 1,
+			"jobId":             jobId,
+			"externalId":        externalId,
+			"resourceName":      resourceName,
+			"resourceType":      resourceType,
+			"resourceStateData": resourceStateData,
+			"resourceData":      resourceData,
+		},
+	}
+	return txn.Update(ctx, jobStmt)
+}
+
+func getLatestResourceVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) {
+	// Fetch latest version for the resource from history table.
+	stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_RESOURCE_HISTORY WHERE ResourceId = @resourceId;`,
+		Params: map[string]interface{}{"resourceId": resourceId},
+	}
+	iter := txn.Query(ctx, stmt)
+	defer iter.Stop()
+	version := spanner.NullInt64{}
+	row, err := iter.Next()
+	if err == iterator.Done || err != nil {
+		return 0, err
+	}
+	if err := row.Columns(&version); err != nil {
+		return 0, err
+	}
+	if version.Valid {
+		return version.Int64, nil
+	}
+	return 0, nil
+}
+
+// Update the state of the SMT resource.
+func UpdateSMTResourceState(ctx context.Context, resourceId, state string) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `UPDATE SMT_RESOURCE SET ResourceStateData = @resourceStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
+			WHERE ResourceId = @resourceId;`,
+			Params: map[string]interface{}{
+				"resourceId":        resourceId,
+				"resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}},
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error updating smt resource state: %v", err)
+	}
+	return nil
+}
+
+// Update the external of the SMT resource.
+func UpdateSMTResourceExternalId(ctx context.Context, resourceId, externalId string) error {
+	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+		jobStmt := spanner.Statement{
+			SQL: `UPDATE SMT_RESOURCE SET ExternalId = @externalId, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
+			WHERE ResourceId = @resourceId;`,
+			Params: map[string]interface{}{
+				"resourceId": resourceId,
+				"externalId": externalId,
+			},
+		}
+		_, err := txn.Update(ctx, jobStmt)
+		if err != nil {
+			return err
+		}
+		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error updating smt resource external id: %v", err)
+	}
+	return nil
+}

From ef228243dcabf1dee188cf6b88852cbfd2d1412a Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:19:27 +0530
Subject: [PATCH 30/35] Added dao

---
 dao/resource_metadata.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dao/resource_metadata.go b/dao/resource_metadata.go
index 1c72b298fd..87e435ae30 100644
--- a/dao/resource_metadata.go
+++ b/dao/resource_metadata.go
@@ -44,7 +44,7 @@ func InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId,
 		if err != nil {
 			return err
 		}
-		// Update the resoruce history table in the same transaction.
+		// Update the resource history table in the same transaction.
 		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
 		if err != nil {
 			return err

From dfe6ac768231a022cdff1ce93634f34ce3925fb4 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:22:56 +0530
Subject: [PATCH 31/35] Add activity interface and one activity

---
 common/constants/constants.go                |  3 +
 reverserepl/activity/IActivity.go            | 21 +++++++
 reverserepl/activity/create_smt_job_entry.go | 59 ++++++++++++++++++++
 3 files changed, 83 insertions(+)
 create mode 100644 reverserepl/activity/IActivity.go
 create mode 100644 reverserepl/activity/create_smt_job_entry.go

diff --git a/common/constants/constants.go b/common/constants/constants.go
index c0eaa87685..31eb0f2a34 100644
--- a/common/constants/constants.go
+++ b/common/constants/constants.go
@@ -101,4 +101,7 @@ const (
 	// Metadata table names
 	SMT_JOB_TABLE      string = "SMT_JOB"
 	SMT_RESOURCE_TABLE string = "SMT_RESOURCE"
+
+	// Reverse Replication
+	REVERSE_REPLICATION_JOB_TYPE string = "reverse-replication"
 )
diff --git a/reverserepl/activity/IActivity.go b/reverserepl/activity/IActivity.go
new file mode 100644
index 0000000000..245ec2e8e1
--- /dev/null
+++ b/reverserepl/activity/IActivity.go
@@ -0,0 +1,21 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import "context"
+
+type Activity interface {
+	Transaction(ctx context.Context) error
+	Compensation(ctx context.Context) error
+}
diff --git a/reverserepl/activity/create_smt_job_entry.go b/reverserepl/activity/create_smt_job_entry.go
new file mode 100644
index 0000000000..9c3d47314c
--- /dev/null
+++ b/reverserepl/activity/create_smt_job_entry.go
@@ -0,0 +1,59 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+
+	"cloud.google.com/go/spanner"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/dao"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+)
+
+type CreateSmtJobEntryInput struct {
+	SmtJobId         string
+	JobName          string
+	SpannerProjectId string
+	InstanceId       string
+	DatabaseId       string
+	JobData          string
+}
+
+type CreateSmtJobEntry struct {
+	Input *CreateSmtJobEntryInput
+}
+
+// This creates an entry in the SMT job table.
+func (p *CreateSmtJobEntry) Transaction(ctx context.Context) error {
+	input := p.Input
+	dialect, err := spanneraccessor.GetDatabaseDialect(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", input.SpannerProjectId, input.InstanceId, input.DatabaseId))
+	if err != nil {
+		return fmt.Errorf("could not fetch database dialect: %v", err)
+	}
+	logger.Log.Debug(fmt.Sprintf("found database dialect: %s", dialect))
+	jobData := spanner.NullJSON{Valid: true, Value: input.JobData}
+	err = dao.InsertSMTJobEntry(ctx, input.SmtJobId, input.JobName, constants.REVERSE_REPLICATION_JOB_TYPE, dialect, input.DatabaseId, jobData)
+	if err != nil {
+		return err
+	}
+	logger.Log.Debug("Created entry SMT Job entry")
+	return nil
+}
+
+func (p *CreateSmtJobEntry) Compensation(ctx context.Context) error {
+	return nil
+}

From 4f36c2aaca4602fa4abb348e3e6a974ed0fe9332 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Thu, 11 Jan 2024 14:09:43 +0530
Subject: [PATCH 32/35] Move interface to top level folder

---
 .../activity => activity}/IActivity.go        |   0
 dao/job_metadata.go                           | 155 ---------------
 dao/resource_metadata.go                      | 177 ------------------
 reverserepl/activity/create_smt_job_entry.go  |   8 +-
 .../activity/create_smt_job_entry_test.go     | 109 +++++++++++
 5 files changed, 114 insertions(+), 335 deletions(-)
 rename {reverserepl/activity => activity}/IActivity.go (100%)
 delete mode 100644 dao/job_metadata.go
 delete mode 100644 dao/resource_metadata.go
 create mode 100644 reverserepl/activity/create_smt_job_entry_test.go

diff --git a/reverserepl/activity/IActivity.go b/activity/IActivity.go
similarity index 100%
rename from reverserepl/activity/IActivity.go
rename to activity/IActivity.go
diff --git a/dao/job_metadata.go b/dao/job_metadata.go
deleted file mode 100644
index 6ea0c6e48e..0000000000
--- a/dao/job_metadata.go
+++ /dev/null
@@ -1,155 +0,0 @@
-// Copyright 2024 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package dao
-
-import (
-	"context"
-	"fmt"
-
-	"cloud.google.com/go/spanner"
-	"google.golang.org/api/iterator"
-)
-
-type StateData struct {
-	State string `json:"state"`
-}
-
-// Insert a job entry into the SMT_JOB table.
-func InsertSMTJobEntry(ctx context.Context, jobId, jobName, jobType, dialect, dbName string, jobData spanner.NullJSON) error {
-	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
-		jobStmt := spanner.Statement{
-			SQL: `INSERT INTO SMT_JOB 
-			(JobId, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt, UpdatedAt)
-			VALUES(
-			 @jobId, @jobName, @jobType, @jobStateData, @jobData, @dialect, @dbName, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP()
-			);`,
-			Params: map[string]interface{}{
-				"jobId":        jobId,
-				"jobName":      jobName,
-				"jobType":      jobType,
-				"jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}},
-				"jobData":      jobData,
-				"dialect":      dialect,
-				"dbName":       dbName,
-			},
-		}
-		_, err := txn.Update(ctx, jobStmt)
-		if err != nil {
-			return err
-		}
-		// Update job history table within the same txn.
-		_, err = updateJobHistoryWithinTxn(ctx, txn, jobId)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("could not insert SMT job entry: %v", err)
-	}
-	return nil
-}
-
-func updateJobHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) {
-	version, err := getLatestJobVersionWithinTxn(ctx, txn, jobId)
-	if err != nil {
-		return 0, fmt.Errorf("error fetching latest job version: %v", err)
-	}
-	// Fetch the newly updated row from SMT_JOB table.
-	stmt := spanner.Statement{SQL: `
-		SELECT 
-			JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName 
-		FROM SMT_JOB WHERE JobId = @jobId;`,
-		Params: map[string]interface{}{"jobId": jobId},
-	}
-	iter := txn.Query(ctx, stmt)
-	defer iter.Stop()
-	var jobName, jobType, dialect, spannerDatabaseName spanner.NullString
-	var jobStateData, jobData spanner.NullJSON
-	row, err := iter.Next()
-	if err == iterator.Done || err != nil {
-		return 0, err
-	}
-	if err := row.Columns(&jobName, &jobType, &jobStateData, &jobData, &dialect, &spannerDatabaseName); err != nil {
-		return 0, fmt.Errorf("error reading smt job row: %v", err)
-	}
-
-	// Insert entry to SMT_JOB_HISTORY table.
-	jobStmt := spanner.Statement{
-		SQL: `INSERT INTO SMT_JOB_HISTORY 
-		(JobId, Version, JobName, JobType, JobStateData, JobData, Dialect, SpannerDatabaseName, CreatedAt)
-		VALUES(
-		 @jobId, @version, @jobName, @jobType, @jobStateData, @jobData, @dialect, @spannerDatabaseName, PENDING_COMMIT_TIMESTAMP()
-		);`,
-		Params: map[string]interface{}{
-			"jobId":               jobId,
-			"version":             version + 1,
-			"jobName":             jobName,
-			"jobType":             jobType,
-			"jobStateData":        jobStateData,
-			"jobData":             jobData,
-			"dialect":             dialect,
-			"spannerDatabaseName": spannerDatabaseName,
-		},
-	}
-	return txn.Update(ctx, jobStmt)
-}
-
-func getLatestJobVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, jobId string) (int64, error) {
-	// Fetch latest version for the job from history table.
-	stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_JOB_HISTORY WHERE JobId = @jobId;`,
-		Params: map[string]interface{}{"jobId": jobId},
-	}
-	iter := txn.Query(ctx, stmt)
-	defer iter.Stop()
-	version := spanner.NullInt64{}
-	row, err := iter.Next()
-	if err == iterator.Done || err != nil {
-		return 0, err
-	}
-	if err := row.Columns(&version); err != nil {
-		return 0, err
-	}
-	if version.Valid {
-		return version.Int64, nil
-	}
-	return 0, nil
-}
-
-// Update the state of the SMT job.
-func UpdateSMTJobState(ctx context.Context, jobId, state string) error {
-	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
-		jobStmt := spanner.Statement{
-			SQL: `UPDATE SMT_JOB SET JobStateData = @jobStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
-			WHERE JobId = @jobId;`,
-			Params: map[string]interface{}{
-				"jobId":        jobId,
-				"jobStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}},
-			},
-		}
-		_, err := txn.Update(ctx, jobStmt)
-		if err != nil {
-			return err
-		}
-		_, err = updateJobHistoryWithinTxn(ctx, txn, jobId)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("error updating smt job state: %v", err)
-	}
-	return nil
-}
diff --git a/dao/resource_metadata.go b/dao/resource_metadata.go
deleted file mode 100644
index 87e435ae30..0000000000
--- a/dao/resource_metadata.go
+++ /dev/null
@@ -1,177 +0,0 @@
-// Copyright 2024 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package dao
-
-import (
-	"context"
-	"fmt"
-
-	"cloud.google.com/go/spanner"
-	"google.golang.org/api/iterator"
-)
-
-// Insert an entry into the SMT_RESOURCE table.
-func InsertSMTResourceEntry(ctx context.Context, resourceId, jobId, externalId, resourceName, resourceType string, resourceData spanner.NullJSON) error {
-	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
-		resourceStmt := spanner.Statement{
-			SQL: `INSERT INTO SMT_RESOURCE
-			(ResourceId, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt, UpdatedAt)
-			VALUES(
-			 @resourceId, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP(), PENDING_COMMIT_TIMESTAMP()
-			);`,
-			Params: map[string]interface{}{
-				"resourceId":        resourceId,
-				"jobId":             jobId,
-				"externalId":        externalId,
-				"resourceName":      resourceName,
-				"resourceType":      resourceType,
-				"resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: "CREATING"}},
-				"resourceData":      resourceData,
-			},
-		}
-		_, err := txn.Update(ctx, resourceStmt)
-		if err != nil {
-			return err
-		}
-		// Update the resource history table in the same transaction.
-		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("error inserting smt resource entry: %v", err)
-	}
-	return nil
-}
-
-func updateResourceHistoryWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) {
-	version, err := getLatestResourceVersionWithinTxn(ctx, txn, resourceId)
-	if err != nil {
-		return 0, fmt.Errorf("error fetching latest resource version: %v", err)
-	}
-	// Fetch the newly updated row from SMT_RESOURCE table.
-	stmt := spanner.Statement{SQL: `
-		SELECT 
-			JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData 
-		FROM SMT_RESOURCE WHERE ResourceId = @resourceId;`,
-		Params: map[string]interface{}{"resourceId": resourceId},
-	}
-	iter := txn.Query(ctx, stmt)
-	defer iter.Stop()
-	var jobId, externalId, resourceName, resourceType spanner.NullString
-	var resourceStateData, resourceData spanner.NullJSON
-	row, err := iter.Next()
-	if err == iterator.Done || err != nil {
-		return 0, err
-	}
-	if err := row.Columns(&jobId, &externalId, &resourceName, &resourceType, &resourceStateData, &resourceData); err != nil {
-		return 0, fmt.Errorf("error reading smt resource row: %v", err)
-	}
-	// Create new entry into the SMT_RESOURCE_HISTORY table.
-	jobStmt := spanner.Statement{
-		SQL: `INSERT INTO SMT_RESOURCE_HISTORY 
-		(ResourceId, Version, JobId, ExternalId, ResourceName, ResourceType, ResourceStateData, ResourceData, CreatedAt)
-		VALUES(
-		 @resourceId, @version, @jobId, @externalId, @resourceName, @resourceType, @resourceStateData, @resourceData, PENDING_COMMIT_TIMESTAMP()
-		);`,
-		Params: map[string]interface{}{
-			"resourceId":        resourceId,
-			"version":           version + 1,
-			"jobId":             jobId,
-			"externalId":        externalId,
-			"resourceName":      resourceName,
-			"resourceType":      resourceType,
-			"resourceStateData": resourceStateData,
-			"resourceData":      resourceData,
-		},
-	}
-	return txn.Update(ctx, jobStmt)
-}
-
-func getLatestResourceVersionWithinTxn(ctx context.Context, txn *spanner.ReadWriteTransaction, resourceId string) (int64, error) {
-	// Fetch latest version for the resource from history table.
-	stmt := spanner.Statement{SQL: `SELECT MAX(Version) FROM SMT_RESOURCE_HISTORY WHERE ResourceId = @resourceId;`,
-		Params: map[string]interface{}{"resourceId": resourceId},
-	}
-	iter := txn.Query(ctx, stmt)
-	defer iter.Stop()
-	version := spanner.NullInt64{}
-	row, err := iter.Next()
-	if err == iterator.Done || err != nil {
-		return 0, err
-	}
-	if err := row.Columns(&version); err != nil {
-		return 0, err
-	}
-	if version.Valid {
-		return version.Int64, nil
-	}
-	return 0, nil
-}
-
-// Update the state of the SMT resource.
-func UpdateSMTResourceState(ctx context.Context, resourceId, state string) error {
-	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
-		jobStmt := spanner.Statement{
-			SQL: `UPDATE SMT_RESOURCE SET ResourceStateData = @resourceStateData, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
-			WHERE ResourceId = @resourceId;`,
-			Params: map[string]interface{}{
-				"resourceId":        resourceId,
-				"resourceStateData": spanner.NullJSON{Valid: true, Value: StateData{State: state}},
-			},
-		}
-		_, err := txn.Update(ctx, jobStmt)
-		if err != nil {
-			return err
-		}
-		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("error updating smt resource state: %v", err)
-	}
-	return nil
-}
-
-// Update the external of the SMT resource.
-func UpdateSMTResourceExternalId(ctx context.Context, resourceId, externalId string) error {
-	_, err := GetClient().ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
-		jobStmt := spanner.Statement{
-			SQL: `UPDATE SMT_RESOURCE SET ExternalId = @externalId, UpdatedAt = PENDING_COMMIT_TIMESTAMP()
-			WHERE ResourceId = @resourceId;`,
-			Params: map[string]interface{}{
-				"resourceId": resourceId,
-				"externalId": externalId,
-			},
-		}
-		_, err := txn.Update(ctx, jobStmt)
-		if err != nil {
-			return err
-		}
-		_, err = updateResourceHistoryWithinTxn(ctx, txn, resourceId)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("error updating smt resource external id: %v", err)
-	}
-	return nil
-}
diff --git a/reverserepl/activity/create_smt_job_entry.go b/reverserepl/activity/create_smt_job_entry.go
index 9c3d47314c..9ef0d60827 100644
--- a/reverserepl/activity/create_smt_job_entry.go
+++ b/reverserepl/activity/create_smt_job_entry.go
@@ -35,18 +35,20 @@ type CreateSmtJobEntryInput struct {
 
 type CreateSmtJobEntry struct {
 	Input *CreateSmtJobEntryInput
+	DAO   dao.DAO
+	SpA   spanneraccessor.SpannerAccessor
 }
 
-// This creates an entry in the SMT job table.
+// This creates a reverse replication entry in the SMT job table.
 func (p *CreateSmtJobEntry) Transaction(ctx context.Context) error {
 	input := p.Input
-	dialect, err := spanneraccessor.GetDatabaseDialect(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", input.SpannerProjectId, input.InstanceId, input.DatabaseId))
+	dialect, err := p.SpA.GetDatabaseDialect(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", input.SpannerProjectId, input.InstanceId, input.DatabaseId))
 	if err != nil {
 		return fmt.Errorf("could not fetch database dialect: %v", err)
 	}
 	logger.Log.Debug(fmt.Sprintf("found database dialect: %s", dialect))
 	jobData := spanner.NullJSON{Valid: true, Value: input.JobData}
-	err = dao.InsertSMTJobEntry(ctx, input.SmtJobId, input.JobName, constants.REVERSE_REPLICATION_JOB_TYPE, dialect, input.DatabaseId, jobData)
+	err = p.DAO.InsertSMTJobEntry(ctx, input.SmtJobId, input.JobName, constants.REVERSE_REPLICATION_JOB_TYPE, dialect, input.DatabaseId, jobData)
 	if err != nil {
 		return err
 	}
diff --git a/reverserepl/activity/create_smt_job_entry_test.go b/reverserepl/activity/create_smt_job_entry_test.go
new file mode 100644
index 0000000000..e28e43c264
--- /dev/null
+++ b/reverserepl/activity/create_smt_job_entry_test.go
@@ -0,0 +1,109 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"cloud.google.com/go/spanner"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/dao"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+)
+
+func init() {
+	logger.Log = zap.NewNop()
+}
+
+func TestMain(m *testing.M) {
+	res := m.Run()
+	os.Exit(res)
+}
+
+type SpannerAccessorMock struct {
+	spanneraccessor.SpannerAccessor
+}
+
+var getDatabaseDialectMock func(ctx context.Context, dbURI string) (string, error)
+
+func (sam *SpannerAccessorMock) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
+	return getDatabaseDialectMock(ctx, dbURI)
+}
+
+type DAOMock struct {
+	dao.DAOImpl
+}
+
+var insertSMTJobEntryMock func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error
+
+func (dao *DAOMock) InsertSMTJobEntry(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error {
+	return insertSMTJobEntryMock(ctx, jobId, jobName, jobType, dialect, dbName, jobData)
+}
+
+func TestCreateSmtJobEntryTransaction(t *testing.T) {
+	testCases := []struct {
+		name                   string
+		getDatabaseDialectMock func(ctx context.Context, dbURI string) (string, error)
+		insertSMTJobEntryMock  func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error
+		expectError            bool
+	}{
+		{
+			name: "No errors",
+			getDatabaseDialectMock: func(ctx context.Context, dbURI string) (string, error) {
+				return "", nil
+			},
+			insertSMTJobEntryMock: func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error {
+				return nil
+			},
+			expectError: false,
+		},
+		{
+			name: "Fetch Dialect error",
+			getDatabaseDialectMock: func(ctx context.Context, dbURI string) (string, error) {
+				return "", fmt.Errorf("test error")
+			},
+			insertSMTJobEntryMock: func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error {
+				return nil
+			},
+			expectError: true,
+		},
+		{
+			name: "Dao error",
+			getDatabaseDialectMock: func(ctx context.Context, dbURI string) (string, error) {
+				return "", nil
+			},
+			insertSMTJobEntryMock: func(ctx context.Context, jobId string, jobName string, jobType string, dialect string, dbName string, jobData spanner.NullJSON) error {
+				return fmt.Errorf("test error")
+			},
+			expectError: true,
+		},
+	}
+	ctx := context.Background()
+	createSmtJobEntry := CreateSmtJobEntry{
+		Input: &CreateSmtJobEntryInput{},
+		DAO:   &DAOMock{},
+		SpA:   &SpannerAccessorMock{},
+	}
+	for _, tc := range testCases {
+		getDatabaseDialectMock = tc.getDatabaseDialectMock
+		insertSMTJobEntryMock = tc.insertSMTJobEntryMock
+		err := createSmtJobEntry.Transaction(ctx)
+		assert.Equal(t, tc.expectError, err != nil)
+	}
+}

From b68e09a92913f1d3448b4b4f47e544f216245128 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:22:56 +0530
Subject: [PATCH 33/35] Add activity interface and one activity

---
 reverserepl/activity/IActivity.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 reverserepl/activity/IActivity.go

diff --git a/reverserepl/activity/IActivity.go b/reverserepl/activity/IActivity.go
new file mode 100644
index 0000000000..245ec2e8e1
--- /dev/null
+++ b/reverserepl/activity/IActivity.go
@@ -0,0 +1,21 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import "context"
+
+type Activity interface {
+	Transaction(ctx context.Context) error
+	Compensation(ctx context.Context) error
+}

From f482cba6f476448604a3f0db8ee64e5d602d7af4 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Tue, 9 Jan 2024 14:23:56 +0530
Subject: [PATCH 34/35] Add create reverse replication workflow

---
 common/constants/constants.go                 |  17 +-
 reverserepl/activity/prepare_change_stream.go |  70 ++++++
 .../activity/prepare_dataflow_reader.go       | 135 ++++++++++
 .../activity/prepare_dataflow_writer.go       | 121 +++++++++
 reverserepl/activity/prepare_gcs_bucket.go    |  75 ++++++
 reverserepl/activity/prepare_metadata_db.go   |  60 +++++
 reverserepl/activity/update_smt_job_entry.go  |  39 +++
 reverserepl/create.go                         | 235 ++++++++++++++++++
 reverserepl/resource/resource.go              |  99 ++++++++
 reverserepl/resource/resourcepb.go            |  37 +++
 reverserepl/types.go                          |  49 ++++
 11 files changed, 936 insertions(+), 1 deletion(-)
 create mode 100644 reverserepl/activity/prepare_change_stream.go
 create mode 100644 reverserepl/activity/prepare_dataflow_reader.go
 create mode 100644 reverserepl/activity/prepare_dataflow_writer.go
 create mode 100644 reverserepl/activity/prepare_gcs_bucket.go
 create mode 100644 reverserepl/activity/prepare_metadata_db.go
 create mode 100644 reverserepl/activity/update_smt_job_entry.go
 create mode 100644 reverserepl/create.go
 create mode 100644 reverserepl/resource/resource.go
 create mode 100644 reverserepl/resource/resourcepb.go
 create mode 100644 reverserepl/types.go

diff --git a/common/constants/constants.go b/common/constants/constants.go
index 31eb0f2a34..99b33b1034 100644
--- a/common/constants/constants.go
+++ b/common/constants/constants.go
@@ -103,5 +103,20 @@ const (
 	SMT_RESOURCE_TABLE string = "SMT_RESOURCE"
 
 	// Reverse Replication
-	REVERSE_REPLICATION_JOB_TYPE string = "reverse-replication"
+	REVERSE_REPLICATION_JOB_TYPE             string = "reverse-replication"
+	REVERSE_REPLICATION_READER_TEMPLATE_PATH string = "gs://aks-test-revrep/templates/flex/Spanner_Change_Streams_to_Sharded_File_Sink" //"gs://dataflow-templates/latest/flex/Spanner_Change_Streams_to_Sharded_File_Sink"
+	REVERSE_REPLICATION_WRITER_TEMPLATE_PATH string = "gs://aks-test-revrep/templates/flex/GCS_to_Sourcedb"                             //"gs://dataflow-templates/latest/flex/GCS_to_Sourcedb"
+
+	// Reverse replication - Reader template.
+	RR_READER_FILTER_NONE  string = "none"
+	RR_READER_FILTER_FWD   string = "forward_migration"
+	RR_READER_REGULAR_MODE string = "regular"
+	RR_READER_RESUME_MODE  string = "resume"
+
+	// Reverse replication - Writer template.
+	RR_WRITER_REGULAR_MODE        string = "regular"
+	RR_WRITER_REPROCESS_MODE      string = "reprocess"
+	RR_WRITER_RESUME_SUCCESS_MODE string = "resumeSuccess"
+	RR_WRITER_RESUME_FAILED_MODE  string = "resumeFailed"
+	RR_WRITER_RESUME_ALL_MODE     string = "resumeAll"
 )
diff --git a/reverserepl/activity/prepare_change_stream.go b/reverserepl/activity/prepare_change_stream.go
new file mode 100644
index 0000000000..6476738ef2
--- /dev/null
+++ b/reverserepl/activity/prepare_change_stream.go
@@ -0,0 +1,70 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource"
+)
+
+type PrepareChangeStreamInput struct {
+	SmtJobId         string
+	ChangeStreamName string
+	DbURI            string
+}
+
+type PrepareChangeStreamOutput struct {
+	Exists                     bool
+	ExistsWithIncorrectOptions bool
+	Created                    bool
+}
+
+type PrepareChangeStream struct {
+	Input  *PrepareChangeStreamInput
+	Output *PrepareChangeStreamOutput
+}
+
+// This checks is a valid change stream exists or not. If not, it creates one on the provided DbURI.
+func (p *PrepareChangeStream) Transaction(ctx context.Context) error {
+	input := p.Input
+	csExists, err := spanneraccessor.CheckIfChangeStreamExists(ctx, input.ChangeStreamName, input.DbURI)
+	if err != nil {
+		return err
+	}
+	if csExists {
+		err = spanneraccessor.ValidateChangeStreamOptions(ctx, input.ChangeStreamName, input.DbURI)
+		if err != nil {
+			p.Output.ExistsWithIncorrectOptions = true
+			return fmt.Errorf("invalid change stream option found: %v", err)
+		}
+		logger.Log.Info(fmt.Sprintf("change stream %s already exists for %s, skipping creation", input.ChangeStreamName, input.DbURI))
+		p.Output.Exists = true
+		return nil
+	}
+	err = resource.CreateChangeStreamSMTResource(ctx, input.SmtJobId, input.ChangeStreamName, input.DbURI)
+	if err != nil {
+		return fmt.Errorf("could not create change stream resource: %v", err)
+	}
+	logger.Log.Info(fmt.Sprintf("Created change stream %s for %s", input.ChangeStreamName, input.DbURI))
+	p.Output.Created = true
+	return nil
+}
+
+func (p *PrepareChangeStream) Compensation(ctx context.Context) error {
+	return nil
+}
diff --git a/reverserepl/activity/prepare_dataflow_reader.go b/reverserepl/activity/prepare_dataflow_reader.go
new file mode 100644
index 0000000000..464a3b8e66
--- /dev/null
+++ b/reverserepl/activity/prepare_dataflow_reader.go
@@ -0,0 +1,135 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+	"slices"
+
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource"
+)
+
+type PrepareDataflowReaderInput struct {
+	SmtJobId             string
+	ChangeStreamName     string
+	InstanceId           string
+	DatabaseId           string
+	SpannerProjectId     string
+	SessionFilePath      string
+	SourceShardsFilePath string
+	MetadataInstance     string
+	MetadataDatabase     string
+	GcsOutputDirectory   string
+	StartTimestamp       string
+	EndTimestamp         string
+	WindowDuration       string
+	FiltrationMode       string
+	MetadataTableSuffix  string
+	SkipDirectoryName    string
+	TuningCfg            string
+	SpannerLocation      string
+}
+
+type PrepareDataflowReaderOutput struct {
+	JobId string
+}
+
+type PrepareDataflowReader struct {
+	Input  *PrepareDataflowReaderInput
+	Output *PrepareDataflowReaderOutput
+}
+
+// Launches the reader dataflow job.
+func (p *PrepareDataflowReader) Transaction(ctx context.Context) error {
+	input := p.Input
+	readerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, input.TuningCfg)
+	if err != nil {
+		return fmt.Errorf("error reading reader tuning config %s: %v", input.TuningCfg, err)
+	}
+	logger.Log.Debug(fmt.Sprintf("readerTuningCfg: %+v", readerTuningCfg))
+	validateUpdateReaderTuningCfg(&readerTuningCfg, input.SpannerProjectId, input.SpannerLocation, input.SmtJobId)
+	logger.Log.Debug(fmt.Sprintf("Updated readerTuningCfg: %+v", readerTuningCfg))
+
+	params := map[string]string{
+		"changeStreamName":     input.ChangeStreamName,
+		"instanceId":           input.InstanceId,
+		"databaseId":           input.DatabaseId,
+		"spannerProjectId":     input.SpannerProjectId,
+		"metadataInstance":     input.MetadataInstance,
+		"metadataDatabase":     input.MetadataDatabase,
+		"gcsOutputDirectory":   input.GcsOutputDirectory,
+		"sessionFilePath":      input.SessionFilePath,
+		"sourceShardsFilePath": input.SourceShardsFilePath,
+		"endTimestamp":         input.EndTimestamp,
+		"windowDuration":       input.WindowDuration,
+		"filtrationMode":       input.FiltrationMode,
+		"metadataTableSuffix":  input.MetadataTableSuffix,
+		"skipDirectoryName":    input.SkipDirectoryName,
+		"startTimestamp":       input.StartTimestamp,
+		"runIdentifier":        input.SmtJobId,
+		"runMode":              constants.RR_READER_REGULAR_MODE,
+	}
+	dfLaunchReq, err := dataflowutils.GetDataflowLaunchRequest(params, readerTuningCfg)
+	if err != nil {
+		return err
+	}
+	dfJobId, err := resource.CreateDataflowSMTResource(ctx, input.SmtJobId, dfLaunchReq)
+	if err != nil {
+		return err
+	}
+	logger.Log.Info(fmt.Sprintf("Launched reader job with id: %s", dfJobId))
+	p.Output.JobId = dfJobId
+	return nil
+}
+
+func (p *PrepareDataflowReader) Compensation(ctx context.Context) error {
+	return nil
+}
+
+func validateUpdateReaderTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, spannerProjectId, spannerLocation, smtJobId string) {
+	if cfg.ProjectId == "" {
+		cfg.ProjectId = spannerProjectId
+	}
+	if cfg.JobName == "" {
+		cfg.JobName = fmt.Sprintf("smt-reader-job-%s", utils.GenerateHashStr())
+	}
+	if cfg.Location == "" {
+		cfg.Location = spannerLocation
+	}
+	if cfg.MaxWorkers == 0 {
+		cfg.MaxWorkers = 50
+	}
+	if cfg.NumWorkers == 0 {
+		cfg.NumWorkers = 5
+	}
+	if cfg.MachineType == "" {
+		cfg.MachineType = "n1-standard-2"
+	}
+	cfg.AdditionalUserLabels["smt-reader-job"] = smtJobId
+	if cfg.GcsTemplatePath == "" {
+		cfg.GcsTemplatePath = constants.REVERSE_REPLICATION_READER_TEMPLATE_PATH
+	}
+	if cfg.AdditionalExperiments == nil {
+		cfg.AdditionalExperiments = []string{"use_runner_v2"}
+	} else if !slices.Contains(cfg.AdditionalExperiments, "use_runner_v2") {
+		cfg.AdditionalExperiments = append(cfg.AdditionalExperiments, "use_runner_v2")
+	}
+	cfg.EnableStreamingEngine = true
+}
diff --git a/reverserepl/activity/prepare_dataflow_writer.go b/reverserepl/activity/prepare_dataflow_writer.go
new file mode 100644
index 0000000000..54f6c81a96
--- /dev/null
+++ b/reverserepl/activity/prepare_dataflow_writer.go
@@ -0,0 +1,121 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource"
+)
+
+type PrepareDataflowWriterInput struct {
+	SmtJobId               string
+	SourceShardsFilePath   string
+	SessionFilePath        string
+	SourceType             string
+	SourceDbTimezoneOffset string
+	TimerInterval          int
+	StartTimestamp         string
+	WindowDuration         string
+	GCSInputDirectoryPath  string
+	SpannerProjectId       string
+	MetadataInstance       string
+	MetadataDatabase       string
+	MetadataTableSuffix    string
+	TuningCfg              string
+	SpannerLocation        string
+}
+
+type PrepareDataflowWriterOutput struct {
+	JobId string
+}
+
+type PrepareDataflowWriter struct {
+	Input  *PrepareDataflowWriterInput
+	Output *PrepareDataflowWriterOutput
+}
+
+// Launches the writer dataflow job.
+func (p *PrepareDataflowWriter) Transaction(ctx context.Context) error {
+	input := p.Input
+	writerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, input.TuningCfg)
+	if err != nil {
+		return fmt.Errorf("error reading writer tuning config %s: %v", input.TuningCfg, err)
+	}
+	logger.Log.Debug(fmt.Sprintf("writerTuningCfg: %+v", writerTuningCfg))
+	validateUpdateWriterTuningCfg(&writerTuningCfg, input.SpannerProjectId, input.SpannerLocation, input.SmtJobId)
+	logger.Log.Debug(fmt.Sprintf("Updated writerTuningCfg: %+v", writerTuningCfg))
+	params := map[string]string{
+		"sourceShardsFilePath":   input.SourceShardsFilePath,
+		"sessionFilePath":        input.SessionFilePath,
+		"sourceType":             input.SourceType,
+		"sourceDbTimezoneOffset": input.SourceDbTimezoneOffset,
+		"timerInterval":          fmt.Sprintf("%v", input.TimerInterval),
+		"windowDuration":         input.WindowDuration,
+		"GCSInputDirectoryPath":  input.GCSInputDirectoryPath,
+		"metadataTableSuffix":    input.MetadataTableSuffix,
+		"spannerProjectId":       input.SpannerProjectId,
+		"metadataInstance":       input.MetadataInstance,
+		"metadataDatabase":       input.MetadataDatabase,
+		"startTimestamp":         input.StartTimestamp,
+		"runIdentifier":          input.SmtJobId,
+		"runMode":                constants.RR_WRITER_REGULAR_MODE,
+	}
+	dfLaunchReq, err := dataflowutils.GetDataflowLaunchRequest(params, writerTuningCfg)
+	if err != nil {
+		return err
+	}
+	dfJobId, err := resource.CreateDataflowSMTResource(ctx, input.SmtJobId, dfLaunchReq)
+	if err != nil {
+		return err
+	}
+	logger.Log.Info(fmt.Sprintf("Launched writer job with id: %s", dfJobId))
+	p.Output.JobId = dfJobId
+	return nil
+}
+
+func (p *PrepareDataflowWriter) Compensation(ctx context.Context) error {
+	return nil
+}
+
+func validateUpdateWriterTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, spannerProjectId, spannerLocation, smtJobId string) {
+	if cfg.ProjectId == "" {
+		cfg.ProjectId = spannerProjectId
+	}
+	if cfg.JobName == "" {
+		cfg.JobName = fmt.Sprintf("smt-writer-job-%s", utils.GenerateHashStr())
+	}
+	if cfg.Location == "" {
+		cfg.Location = spannerLocation
+	}
+	if cfg.MaxWorkers == 0 {
+		cfg.MaxWorkers = 50
+	}
+	if cfg.NumWorkers == 0 {
+		cfg.NumWorkers = 5
+	}
+	if cfg.MachineType == "" {
+		cfg.MachineType = "n1-standard-2"
+	}
+	cfg.AdditionalUserLabels["smt-writer-job"] = smtJobId
+	if cfg.GcsTemplatePath == "" {
+		cfg.GcsTemplatePath = constants.REVERSE_REPLICATION_WRITER_TEMPLATE_PATH
+	}
+}
diff --git a/reverserepl/activity/prepare_gcs_bucket.go b/reverserepl/activity/prepare_gcs_bucket.go
new file mode 100644
index 0000000000..dc6538dd26
--- /dev/null
+++ b/reverserepl/activity/prepare_gcs_bucket.go
@@ -0,0 +1,75 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource"
+)
+
+type PrepareGcsBucketInput struct {
+	SmtJobId               string
+	SmtBucketName          string
+	SpannerProjectId       string
+	SpannerLocation        string
+	SessionFilePath        string
+	SourceConnectionConfig string
+	IsSMTBucketRequired    bool
+}
+
+type PrepareGcsBucketOutput struct {
+	SessionFilePath        string
+	SourceConnectionConfig string
+}
+
+type PrepareGcsBucket struct {
+	Input *PrepareGcsBucketInput
+}
+
+// This creates a GCS bucket if based on flag input. It subsequently uploads local files to the bucket.
+func (p *PrepareGcsBucket) Transaction(ctx context.Context) error {
+	input := p.Input
+	if input.IsSMTBucketRequired {
+		err := resource.CreateBucketSMTResource(ctx, input.SmtJobId, input.SmtBucketName, input.SpannerProjectId, input.SpannerLocation, nil, 45)
+		if err != nil {
+			return err
+		}
+		logger.Log.Info(fmt.Sprintf("Created bucket: %s", input.SmtBucketName))
+		if !strings.HasPrefix(input.SessionFilePath, constants.GCS_FILE_PREFIX) {
+			err := storageaccessor.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "session.json", input.SessionFilePath)
+			if err != nil {
+				return fmt.Errorf("could not upload session file to GCS: %v", err)
+			}
+			logger.Log.Debug(fmt.Sprintf("Uploaded local session file: %s to bucket %s", input.SessionFilePath, input.SmtBucketName))
+		}
+		if !strings.HasPrefix(input.SourceConnectionConfig, constants.GCS_FILE_PREFIX) {
+			err := storageaccessor.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "source-connection-config.json", input.SourceConnectionConfig)
+			if err != nil {
+				return fmt.Errorf("could not upload source connection config file to GCS: %v", err)
+			}
+			logger.Log.Debug(fmt.Sprintf("Uploaded local source connection config : %s to bucket %s", input.SourceConnectionConfig, input.SmtBucketName))
+		}
+	}
+	return nil
+}
+
+func (p *PrepareGcsBucket) Compensation(ctx context.Context) error {
+	return nil
+}
diff --git a/reverserepl/activity/prepare_metadata_db.go b/reverserepl/activity/prepare_metadata_db.go
new file mode 100644
index 0000000000..052e11319d
--- /dev/null
+++ b/reverserepl/activity/prepare_metadata_db.go
@@ -0,0 +1,60 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+	"fmt"
+
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	resource "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/resource"
+)
+
+type PrepareMetadataDbInput struct {
+	SmtJobId string
+	DbURI    string
+}
+
+type PrepareMetadataDbOutput struct {
+	Exists  bool
+	Created bool
+}
+
+type PrepareMetadataDb struct {
+	Input  *PrepareMetadataDbInput
+	Output *PrepareMetadataDbOutput
+}
+
+// Creates a metadata db for reverse replication if one is not already present.
+func (p *PrepareMetadataDb) Transaction(ctx context.Context) error {
+	input := p.Input
+	dbExists, err := spanneraccessor.CheckExistingDb(ctx, input.DbURI)
+	if err != nil {
+		return fmt.Errorf("error checking existing db: %v", err)
+	}
+	if dbExists {
+		logger.Log.Info(fmt.Sprintf("reverse replication metadata db %s already exists, skipping creation", input.DbURI))
+		p.Output.Exists = true
+		return nil
+	}
+	resource.CreateMetadataDbSMTResource(ctx, input.SmtJobId, input.DbURI)
+	logger.Log.Info(fmt.Sprintf("Created reverse replication metadata db %s", input.DbURI))
+	p.Output.Created = true
+	return nil
+}
+
+func (p *PrepareMetadataDb) Compensation(ctx context.Context) error {
+	return nil
+}
diff --git a/reverserepl/activity/update_smt_job_entry.go b/reverserepl/activity/update_smt_job_entry.go
new file mode 100644
index 0000000000..4c4b81bb72
--- /dev/null
+++ b/reverserepl/activity/update_smt_job_entry.go
@@ -0,0 +1,39 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package activity
+
+import (
+	"context"
+
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/dao"
+)
+
+type UpdateSmtJobEntryInput struct {
+	SmtJobId string
+	State    string
+}
+
+type UpdateSmtJobEntry struct {
+	Input *UpdateSmtJobEntryInput
+}
+
+// Updates the state of an smt job entry.
+func (p *UpdateSmtJobEntry) Transaction(ctx context.Context) error {
+	input := p.Input
+	return dao.UpdateSMTJobState(ctx, input.SmtJobId, input.State)
+}
+
+func (p *UpdateSmtJobEntry) Compensation(ctx context.Context) error {
+	return nil
+}
diff --git a/reverserepl/create.go b/reverserepl/create.go
new file mode 100644
index 0000000000..4e5249389a
--- /dev/null
+++ b/reverserepl/create.go
@@ -0,0 +1,235 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package reverserepl
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/activity"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/dao"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/logger"
+	rractivity "github.com/GoogleCloudPlatform/spanner-migration-tool/reverserepl/activity"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/webv2/helpers"
+)
+
+func validateAndUpdateJobData(ctx context.Context, request *JobData, uuid string) (err error) {
+	request.IsSMTBucketRequired = true
+	request.SmtBucketName = fmt.Sprintf("smt-rr-gcs-%s", uuid)
+	if strings.HasPrefix(request.SessionFilePath, constants.GCS_FILE_PREFIX) && strings.HasPrefix(request.SourceConnectionConfig, constants.GCS_FILE_PREFIX) && request.GcsDataDirectory != "" {
+		request.IsSMTBucketRequired = false
+		request.SmtBucketName = ""
+	}
+	if request.InstanceId == "" {
+		return fmt.Errorf("found empty InstanceId which is a required parameter")
+	}
+	if request.DatabaseId == "" {
+		return fmt.Errorf("found empty DatabaseId which is a required parameter")
+	}
+	if request.SessionFilePath == "" {
+		return fmt.Errorf("found empty SessionFilePath which is a required parameter")
+	} else if !strings.HasPrefix(request.SessionFilePath, constants.GCS_FILE_PREFIX) {
+		request.SessionFileGcsPath = fmt.Sprintf("%s%s/session.json", constants.GCS_FILE_PREFIX, request.SmtBucketName)
+	} else {
+		request.SessionFileGcsPath = request.SessionFilePath
+	}
+	if request.SourceConnectionConfig == "" {
+		return fmt.Errorf("found empty SourceConnectionConfig which is a required parameter")
+	} else if !strings.HasPrefix(request.SourceConnectionConfig, constants.GCS_FILE_PREFIX) {
+		request.SourceConnectionConfigGcsPath = fmt.Sprintf("%s%s/source-connection-config.json", constants.GCS_FILE_PREFIX, request.SmtBucketName)
+	} else {
+		request.SourceConnectionConfigGcsPath = request.SourceConnectionConfig
+	}
+	if request.SpannerProjectId == "" {
+		return fmt.Errorf("found empty SpannerProjectId which is a required parameter")
+	}
+	if request.JobName == "" {
+		request.JobName = fmt.Sprintf("smt-job-%s", uuid)
+	}
+	if request.SourceType == "" {
+		request.SourceType = constants.MYSQL
+	}
+	if request.SourceType != constants.MYSQL {
+		return fmt.Errorf("%s is not a valid source type for reverse replication. Only supported source type is mysql", request.SourceType)
+	}
+	if request.MetadataInstance == "" {
+		request.MetadataInstance = request.InstanceId
+	}
+	if request.MetadataDatabase == "" {
+		request.MetadataDatabase = fmt.Sprintf("smt-rr-metadata-%s", uuid)
+	}
+	if request.GcsDataDirectory == "" {
+		request.GcsDataDirectory = fmt.Sprintf("gs://smt-rr-gcs-%s/reverse-replication/data", uuid)
+	} else if !strings.HasPrefix(request.GcsDataDirectory, constants.GCS_FILE_PREFIX) {
+		return fmt.Errorf("invalid gcs path for GcsDataDirectory: %s", request.GcsDataDirectory)
+	}
+	if request.ChangeStreamName == "" {
+		request.ChangeStreamName = fmt.Sprintf("smt-rr-cs-%s", uuid)
+	}
+	if request.FiltrationMode == "" {
+		request.FiltrationMode = constants.RR_READER_FILTER_FWD
+	} else if !slices.Contains([]string{constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE}, request.FiltrationMode) {
+		return fmt.Errorf("found filtrationMode %s, only allowed values are [%s, %s]", request.FiltrationMode, constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE)
+	}
+	if request.TimerInterval < 1 {
+		request.TimerInterval = 1
+	}
+	if request.WindowDuration == "" {
+		request.WindowDuration = "10s"
+	}
+
+	// Replace '-' with '_' since hyphens are not allowed in cs names.
+	request.ChangeStreamName = strings.Replace(request.ChangeStreamName, "-", "_", -1)
+
+	request.SpannerLocation, err = spanneraccessor.GetSpannerLeaderLocation(ctx, fmt.Sprintf("projects/%s/instances/%s", request.SpannerProjectId, request.InstanceId))
+	return err
+}
+
+// CreateWorkflows sets up the data flow job and required resources for a reverse replication pipeline.
+func CreateWorkflow(ctx context.Context, request JobData) error {
+	// Move to initialization to CLI layer.
+	_ = logger.InitializeLogger("DEBUG")
+	defer logger.Log.Sync()
+
+	logger.Log.Info("Creating reverse replication pipeline.")
+	logger.Log.Debug(fmt.Sprintf("Received Create Reverse Replication job request: %+v\n", request))
+	uuid := utils.GenerateHashStr()
+	err := validateAndUpdateJobData(ctx, &request, uuid)
+	if err != nil {
+		return fmt.Errorf("error in validateCreateRequest: %v", err)
+	}
+	logger.Log.Debug(fmt.Sprintf("Updated job request: %+v\n", request))
+
+	// Check or create the internal metadata database for all flows.
+	helpers.CheckOrCreateMetadataDb(request.SpannerProjectId, request.InstanceId)
+	smtMetadataDBURI := helpers.GetSpannerUri(request.SpannerProjectId, request.InstanceId)
+	// Init dao client.
+	_, err = dao.GetOrCreateClient(ctx, smtMetadataDBURI)
+	if err != nil {
+		return fmt.Errorf("error starting dao client: %v", err)
+	}
+
+	smtJobId := fmt.Sprintf("smt-job-%s", uuid)
+	b, err := json.Marshal(request)
+	if err != nil {
+		return fmt.Errorf("error converting job data to string: %v", err)
+	}
+	jobData := string(b)
+	activities := []activity.Activity{
+		&rractivity.CreateSmtJobEntry{
+			Input: &rractivity.CreateSmtJobEntryInput{
+				SmtJobId:         smtJobId,
+				JobName:          request.JobName,
+				SpannerProjectId: request.SpannerProjectId,
+				InstanceId:       request.InstanceId,
+				DatabaseId:       request.DatabaseId,
+				JobData:          jobData,
+			},
+		},
+		&rractivity.PrepareGcsBucket{
+			Input: &rractivity.PrepareGcsBucketInput{
+				SmtJobId:               smtJobId,
+				SmtBucketName:          request.SmtBucketName,
+				SpannerProjectId:       request.SpannerProjectId,
+				SpannerLocation:        request.SpannerLocation,
+				SessionFilePath:        request.SessionFilePath,
+				SourceConnectionConfig: request.SourceConnectionConfig,
+				IsSMTBucketRequired:    request.IsSMTBucketRequired,
+			},
+		},
+		&rractivity.PrepareChangeStream{
+			Input: &rractivity.PrepareChangeStreamInput{
+				SmtJobId:         smtJobId,
+				ChangeStreamName: request.ChangeStreamName,
+				DbURI:            fmt.Sprintf("projects/%s/instances/%s/databases/%s", request.SpannerProjectId, request.InstanceId, request.DatabaseId),
+			},
+			Output: &rractivity.PrepareChangeStreamOutput{},
+		},
+		&rractivity.PrepareMetadataDb{
+			Input: &rractivity.PrepareMetadataDbInput{
+				SmtJobId: smtJobId,
+				DbURI:    fmt.Sprintf("projects/%s/instances/%s/databases/%s", request.SpannerProjectId, request.MetadataInstance, request.MetadataDatabase),
+			},
+			Output: &rractivity.PrepareMetadataDbOutput{},
+		},
+		&rractivity.PrepareDataflowReader{
+			Input: &rractivity.PrepareDataflowReaderInput{
+				SmtJobId:             smtJobId,
+				ChangeStreamName:     request.ChangeStreamName,
+				InstanceId:           request.InstanceId,
+				DatabaseId:           request.DatabaseId,
+				SpannerProjectId:     request.SpannerProjectId,
+				SessionFilePath:      request.SessionFileGcsPath,
+				SourceShardsFilePath: request.SourceConnectionConfigGcsPath,
+				MetadataInstance:     request.MetadataInstance,
+				MetadataDatabase:     request.MetadataDatabase,
+				GcsOutputDirectory:   request.GcsDataDirectory,
+				StartTimestamp:       request.StartTimestamp,
+				EndTimestamp:         request.EndTimestamp,
+				WindowDuration:       request.WindowDuration,
+				FiltrationMode:       request.FiltrationMode,
+				MetadataTableSuffix:  request.MetadataTableSuffix,
+				SkipDirectoryName:    request.SkipDirectoryName,
+				TuningCfg:            request.ReaderCfg,
+				SpannerLocation:      request.SpannerLocation,
+			},
+			Output: &rractivity.PrepareDataflowReaderOutput{},
+		},
+		&rractivity.PrepareDataflowWriter{
+			Input: &rractivity.PrepareDataflowWriterInput{
+				SmtJobId:               smtJobId,
+				SourceShardsFilePath:   request.SourceConnectionConfigGcsPath,
+				SessionFilePath:        request.SessionFileGcsPath,
+				SourceType:             request.SourceType,
+				SourceDbTimezoneOffset: request.SourceDbTimezoneOffset,
+				TimerInterval:          request.TimerInterval,
+				StartTimestamp:         request.StartTimestamp,
+				WindowDuration:         request.WindowDuration,
+				GCSInputDirectoryPath:  request.GcsDataDirectory,
+				SpannerProjectId:       request.SpannerProjectId,
+				MetadataInstance:       request.MetadataInstance,
+				MetadataDatabase:       request.MetadataDatabase,
+				MetadataTableSuffix:    request.MetadataTableSuffix,
+				TuningCfg:              request.WriterCfg,
+				SpannerLocation:        request.SpannerLocation,
+			},
+			Output: &rractivity.PrepareDataflowWriterOutput{},
+		},
+		&rractivity.UpdateSmtJobEntry{
+			Input: &rractivity.UpdateSmtJobEntryInput{
+				SmtJobId: smtJobId,
+				State:    "RUNNING",
+			},
+		},
+	}
+	for i, activity := range activities {
+		if err := activity.Transaction(ctx); err != nil {
+			// If a local transaction fails, execute the compensating actions for all previous steps
+			// for i := len(s.Steps) - 1; i >= 0; i-- {
+			//     if err := s.Steps[i].Compensate(); err != nil {
+			//         return errors.New(fmt.Sprintf("failed to compensate for step %d: %v", i, err))
+			//     }
+			// }
+			return fmt.Errorf("error executing activity #%d: %v", i, err)
+		}
+	}
+	logger.Log.Info("Successfully launched reverse replication pipeline.")
+	return nil
+}
diff --git a/reverserepl/resource/resource.go b/reverserepl/resource/resource.go
new file mode 100644
index 0000000000..a7f342cdb6
--- /dev/null
+++ b/reverserepl/resource/resource.go
@@ -0,0 +1,99 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package resource
+
+import (
+	"context"
+	"fmt"
+
+	"cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
+	"cloud.google.com/go/spanner"
+	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
+	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/dao"
+)
+
+func CreateChangeStreamSMTResource(ctx context.Context, smtJobId, changeStreamName, dbURI string) error {
+	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
+	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_ChangeStream{DbURI: dbURI}}
+	err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, changeStreamName, changeStreamName, "change-stream", resourceData)
+	if err != nil {
+		return fmt.Errorf("error inserting SMT change stream resource: %v", err)
+	}
+	err = spanneraccessor.CreateChangeStream(ctx, changeStreamName, dbURI)
+	if err != nil {
+		return fmt.Errorf("error in change stream creation: %v", err)
+	}
+	return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
+}
+
+func CreateMetadataDbSMTResource(ctx context.Context, smtJobId, dbURI string) error {
+	_, _, dbName := utils.ParseDbURI(dbURI)
+	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
+	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_MetadataDb{DbURI: dbURI}}
+	err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, dbName, dbName, "rr-metadata-db", resourceData)
+	if err != nil {
+		return fmt.Errorf("error inserting SMT metadata db resource: %v", err)
+	}
+	err = spanneraccessor.CreateEmptyDatabase(ctx, dbURI)
+	if err != nil {
+		return fmt.Errorf("error creating db: %v", err)
+	}
+	return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
+}
+
+func CreateBucketSMTResource(ctx context.Context, smtJobId, bucketName, projectId, location string, matchesPrefix []string, ttl int64) error {
+	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
+	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_GCSBucket{
+		Name:          bucketName,
+		ProjectId:     projectId,
+		Location:      location,
+		MatchesPrefix: matchesPrefix,
+		Ttl:           ttl,
+	}}
+	err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, bucketName, bucketName, "gcs-bucket", resourceData)
+	if err != nil {
+		return fmt.Errorf("error inserting SMT bucket resource: %v", err)
+	}
+	err = storageaccessor.CreateGCSBucketWithLifecycle(ctx, bucketName, projectId, location, matchesPrefix, ttl)
+	if err != nil {
+		return fmt.Errorf("error in bucket creation: %v", err)
+	}
+	return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
+}
+
+func CreateDataflowSMTResource(ctx context.Context, smtJobId string, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (string, error) {
+	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
+	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_Dataflow{LaunchRequest: launchRequest, EquivalentGcloudCmd: dataflowutils.GetGcloudDataflowCommand(launchRequest)}}
+	err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, "", launchRequest.LaunchParameter.JobName, "dataflow", resourceData)
+	if err != nil {
+		return "", fmt.Errorf("error inserting SMT dataflow resource: %v", err)
+	}
+	response, err := dataflowaccessor.LaunchDataflowJob(ctx, launchRequest)
+	if err != nil {
+		return "", fmt.Errorf("error in launching dataflow job: %v", err)
+	}
+	err = dao.UpdateSMTResourceExternalId(ctx, resourceId, response.Job.Id)
+	if err != nil {
+		return "", fmt.Errorf("error updating external id for dataflow job: %v", err)
+	}
+	err = dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
+	if err != nil {
+		return "", fmt.Errorf("error updating state for dataflow job: %v", err)
+	}
+	return response.Job.Id, nil
+}
diff --git a/reverserepl/resource/resourcepb.go b/reverserepl/resource/resourcepb.go
new file mode 100644
index 0000000000..08404f2aee
--- /dev/null
+++ b/reverserepl/resource/resourcepb.go
@@ -0,0 +1,37 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package resource
+
+import "cloud.google.com/go/dataflow/apiv1beta3/dataflowpb"
+
+type ResourceData_ChangeStream struct {
+	DbURI string `json:"dbURI"`
+}
+
+type ResourceData_MetadataDb struct {
+	DbURI string `json:"dbURI"`
+}
+
+type ResourceData_GCSBucket struct {
+	Name          string   `json:"name"`
+	ProjectId     string   `json:"projectId"`
+	Location      string   `json:"location"`
+	Ttl           int64    `json:"ttl"`
+	MatchesPrefix []string `json:"matchesPrefix"`
+}
+
+type ResourceData_Dataflow struct {
+	LaunchRequest       *dataflowpb.LaunchFlexTemplateRequest `json:"launchRequest"`
+	EquivalentGcloudCmd string                                `json:"equivalentGcloudCmd"`
+}
diff --git a/reverserepl/types.go b/reverserepl/types.go
new file mode 100644
index 0000000000..ffc9fc283d
--- /dev/null
+++ b/reverserepl/types.go
@@ -0,0 +1,49 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package reverserepl
+
+type JobData struct {
+	// Required parameters.
+	InstanceId             string `json:"instanceId"`
+	DatabaseId             string `json:"databaseId"`
+	SessionFilePath        string `json:"sessionFilePath"`
+	SourceConnectionConfig string `json:"sourceConnectionConfig"`
+	SpannerProjectId       string `json:"spannerProjectId"`
+	// Optional parameters.
+	JobName                string `json:"jobName"`
+	SourceType             string `json:"sourceType"`
+	MetadataInstance       string `json:"metadataInstance"`
+	MetadataDatabase       string `json:"metadataDatabase"`
+	GcsDataDirectory       string `json:"gcsDataDirectory"`
+	ChangeStreamName       string `json:"changeStreamName"`
+	StartTimestamp         string `json:"startTimestamp"`
+	EndTimestamp           string `json:"endTimestamp"`
+	WindowDuration         string `json:"windowDuration"`
+	FiltrationMode         string `json:"filtrationMode"`
+	SourceDbTimezoneOffset string `json:"sourceDbTimezoneOffset"`
+	TimerInterval          int    `json:"timerInterval"`
+	MetadataTableSuffix    string `json:"metadataTableSuffix"`
+	SkipDirectoryName      string `json:"skipDirectoryName"`
+	ReaderCfg              string `json:"readerCfg"`
+	WriterCfg              string `json:"writerCfg"`
+	// SMT generated - These fields are for use internally by SMT. These should not be configured when passing input.
+	IsSMTBucketRequired bool   `json:"isSMTBucketRequired"`
+	SmtBucketName       string `json:"smtBucketName"`
+	// Location of Spanner leader.
+	SpannerLocation string `json:"spannerLocation"`
+	// GCS location of session file path.
+	SessionFileGcsPath string `json:"sessionFileGcsPath"`
+	// GCS location of source connection config.
+	SourceConnectionConfigGcsPath string `json:"sourceConnectionConfigGcsPath"`
+}

From 99411744ccdbaf5263d5da091ff216dfaa543047 Mon Sep 17 00:00:00 2001
From: Deep1998 <deepchowdhury1998@gmail.com>
Date: Mon, 22 Jan 2024 16:57:53 +0530
Subject: [PATCH 35/35] Switch slices to utils

---
 common/constants/constants.go                 |  4 +-
 common/utils/utils.go                         | 11 +++-
 reverserepl/activity/prepare_change_stream.go |  7 ++-
 .../activity/prepare_dataflow_reader.go       | 55 +++++++++++--------
 .../activity/prepare_dataflow_writer.go       | 12 ++--
 reverserepl/activity/prepare_gcs_bucket.go    |  7 ++-
 reverserepl/activity/prepare_metadata_db.go   |  5 +-
 reverserepl/create.go                         | 50 ++++++++++-------
 reverserepl/resource/resource.go              | 16 +++---
 reverserepl/types.go                          | 34 ++++++------
 10 files changed, 120 insertions(+), 81 deletions(-)

diff --git a/common/constants/constants.go b/common/constants/constants.go
index 99b33b1034..11118fc477 100644
--- a/common/constants/constants.go
+++ b/common/constants/constants.go
@@ -104,8 +104,8 @@ const (
 
 	// Reverse Replication
 	REVERSE_REPLICATION_JOB_TYPE             string = "reverse-replication"
-	REVERSE_REPLICATION_READER_TEMPLATE_PATH string = "gs://aks-test-revrep/templates/flex/Spanner_Change_Streams_to_Sharded_File_Sink" //"gs://dataflow-templates/latest/flex/Spanner_Change_Streams_to_Sharded_File_Sink"
-	REVERSE_REPLICATION_WRITER_TEMPLATE_PATH string = "gs://aks-test-revrep/templates/flex/GCS_to_Sourcedb"                             //"gs://dataflow-templates/latest/flex/GCS_to_Sourcedb"
+	REVERSE_REPLICATION_READER_TEMPLATE_PATH string = "gs://dataflow-templates-us-central2/2024-01-09-00_RC01/flex/Spanner_Change_Streams_to_Sharded_File_Sink"
+	REVERSE_REPLICATION_WRITER_TEMPLATE_PATH string = "gs://dataflow-templates-us-central2/2024-01-09-00_RC01/flex/GCS_to_Sourcedb"
 
 	// Reverse replication - Reader template.
 	RR_READER_FILTER_NONE  string = "none"
diff --git a/common/utils/utils.go b/common/utils/utils.go
index 932da34f11..6fc2569890 100644
--- a/common/utils/utils.go
+++ b/common/utils/utils.go
@@ -580,7 +580,7 @@ func CompareSchema(sessionFileConv, actualSpannerConv *internal.Conv) error {
 			} else {
 				if sessionColDef.Name != spannerColDef.Name ||
 					sessionColDef.T.IsArray != spannerColDef.T.IsArray || sessionColDef.T.Len != spannerColDef.T.Len || sessionColDef.T.Name != spannerColDef.T.Name || sessionColDef.NotNull != spannerColDef.NotNull {
-						return fmt.Errorf("column detail for table %v don't match: session column: %v, spanner column: %v", sessionTable.Name, sessionColDef, spannerColDef)
+					return fmt.Errorf("column detail for table %v don't match: session column: %v, spanner column: %v", sessionTable.Name, sessionColDef, spannerColDef)
 				}
 			}
 		}
@@ -637,3 +637,12 @@ func FindInPrimaryKey(id string, primaryKeys []ddl.IndexKey) bool {
 	}
 	return false
 }
+
+func Contains[S ~[]E, E comparable](s S, v E) bool {
+	for i := range s {
+		if v == s[i] {
+			return true
+		}
+	}
+	return false
+}
diff --git a/reverserepl/activity/prepare_change_stream.go b/reverserepl/activity/prepare_change_stream.go
index 6476738ef2..368fcc2606 100644
--- a/reverserepl/activity/prepare_change_stream.go
+++ b/reverserepl/activity/prepare_change_stream.go
@@ -37,17 +37,18 @@ type PrepareChangeStreamOutput struct {
 type PrepareChangeStream struct {
 	Input  *PrepareChangeStreamInput
 	Output *PrepareChangeStreamOutput
+	SpA    spanneraccessor.SpannerAccessor
 }
 
 // This checks is a valid change stream exists or not. If not, it creates one on the provided DbURI.
 func (p *PrepareChangeStream) Transaction(ctx context.Context) error {
 	input := p.Input
-	csExists, err := spanneraccessor.CheckIfChangeStreamExists(ctx, input.ChangeStreamName, input.DbURI)
+	csExists, err := p.SpA.CheckIfChangeStreamExists(ctx, input.ChangeStreamName, input.DbURI)
 	if err != nil {
 		return err
 	}
 	if csExists {
-		err = spanneraccessor.ValidateChangeStreamOptions(ctx, input.ChangeStreamName, input.DbURI)
+		err = p.SpA.ValidateChangeStreamOptions(ctx, input.ChangeStreamName, input.DbURI)
 		if err != nil {
 			p.Output.ExistsWithIncorrectOptions = true
 			return fmt.Errorf("invalid change stream option found: %v", err)
@@ -56,7 +57,7 @@ func (p *PrepareChangeStream) Transaction(ctx context.Context) error {
 		p.Output.Exists = true
 		return nil
 	}
-	err = resource.CreateChangeStreamSMTResource(ctx, input.SmtJobId, input.ChangeStreamName, input.DbURI)
+	err = resource.CreateChangeStreamSMTResource(ctx, p.SpA, input.SmtJobId, input.ChangeStreamName, input.DbURI)
 	if err != nil {
 		return fmt.Errorf("could not create change stream resource: %v", err)
 	}
diff --git a/reverserepl/activity/prepare_dataflow_reader.go b/reverserepl/activity/prepare_dataflow_reader.go
index 464a3b8e66..70880952c1 100644
--- a/reverserepl/activity/prepare_dataflow_reader.go
+++ b/reverserepl/activity/prepare_dataflow_reader.go
@@ -19,6 +19,7 @@ import (
 	"slices"
 
 	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -27,24 +28,26 @@ import (
 )
 
 type PrepareDataflowReaderInput struct {
-	SmtJobId             string
-	ChangeStreamName     string
-	InstanceId           string
-	DatabaseId           string
-	SpannerProjectId     string
-	SessionFilePath      string
-	SourceShardsFilePath string
-	MetadataInstance     string
-	MetadataDatabase     string
-	GcsOutputDirectory   string
-	StartTimestamp       string
-	EndTimestamp         string
-	WindowDuration       string
-	FiltrationMode       string
-	MetadataTableSuffix  string
-	SkipDirectoryName    string
-	TuningCfg            string
-	SpannerLocation      string
+	SmtJobId                string
+	ChangeStreamName        string
+	InstanceId              string
+	DatabaseId              string
+	SpannerProjectId        string
+	SessionFilePath         string
+	SourceShardsFilePath    string
+	MetadataInstance        string
+	MetadataDatabase        string
+	GcsOutputDirectory      string
+	StartTimestamp          string
+	EndTimestamp            string
+	WindowDuration          string
+	FiltrationMode          string
+	MetadataTableSuffix     string
+	SkipDirectoryName       string
+	ShardingCustomJarPath   string
+	ShardingCustomClassName string
+	TuningCfg               string
+	SpannerLocation         string
 }
 
 type PrepareDataflowReaderOutput struct {
@@ -54,12 +57,14 @@ type PrepareDataflowReaderOutput struct {
 type PrepareDataflowReader struct {
 	Input  *PrepareDataflowReaderInput
 	Output *PrepareDataflowReaderOutput
+	DfA    dataflowaccessor.DataflowAccessor
+	SA     storageaccessor.StorageAccessor
 }
 
 // Launches the reader dataflow job.
 func (p *PrepareDataflowReader) Transaction(ctx context.Context) error {
 	input := p.Input
-	readerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, input.TuningCfg)
+	readerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, p.SA, input.TuningCfg)
 	if err != nil {
 		return fmt.Errorf("error reading reader tuning config %s: %v", input.TuningCfg, err)
 	}
@@ -86,15 +91,21 @@ func (p *PrepareDataflowReader) Transaction(ctx context.Context) error {
 		"runIdentifier":        input.SmtJobId,
 		"runMode":              constants.RR_READER_REGULAR_MODE,
 	}
+	// Cannot send empty strings since the template expects GCS file paths.
+	if input.ShardingCustomJarPath != "" {
+		params["shardingCustomJarPath"] = input.ShardingCustomJarPath
+		params["shardingCustomClassName"] = input.ShardingCustomClassName
+	}
 	dfLaunchReq, err := dataflowutils.GetDataflowLaunchRequest(params, readerTuningCfg)
 	if err != nil {
 		return err
 	}
-	dfJobId, err := resource.CreateDataflowSMTResource(ctx, input.SmtJobId, dfLaunchReq)
+	dfJobId, err := resource.CreateDataflowSMTResource(ctx, p.DfA, input.SmtJobId, dfLaunchReq)
 	if err != nil {
 		return err
 	}
 	logger.Log.Info(fmt.Sprintf("Launched reader job with id: %s", dfJobId))
+	logger.Log.Info(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", dfLaunchReq.LaunchParameter.JobName, dataflowutils.GetGcloudDataflowCommand(dfLaunchReq)))
 	p.Output.JobId = dfJobId
 	return nil
 }
@@ -108,7 +119,7 @@ func validateUpdateReaderTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, s
 		cfg.ProjectId = spannerProjectId
 	}
 	if cfg.JobName == "" {
-		cfg.JobName = fmt.Sprintf("smt-reader-job-%s", utils.GenerateHashStr())
+		cfg.JobName = fmt.Sprintf("smt-reverse-replication-reader-%s", utils.GenerateHashStr())
 	}
 	if cfg.Location == "" {
 		cfg.Location = spannerLocation
@@ -122,7 +133,7 @@ func validateUpdateReaderTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, s
 	if cfg.MachineType == "" {
 		cfg.MachineType = "n1-standard-2"
 	}
-	cfg.AdditionalUserLabels["smt-reader-job"] = smtJobId
+	cfg.AdditionalUserLabels["smt-reverse-replication-reader"] = smtJobId
 	if cfg.GcsTemplatePath == "" {
 		cfg.GcsTemplatePath = constants.REVERSE_REPLICATION_READER_TEMPLATE_PATH
 	}
diff --git a/reverserepl/activity/prepare_dataflow_writer.go b/reverserepl/activity/prepare_dataflow_writer.go
index 54f6c81a96..22c1bbe26f 100644
--- a/reverserepl/activity/prepare_dataflow_writer.go
+++ b/reverserepl/activity/prepare_dataflow_writer.go
@@ -18,6 +18,7 @@ import (
 	"fmt"
 
 	dataflowaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/dataflow"
+	storageaccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/storage"
 	dataflowutils "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/utils/dataflow"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/utils"
@@ -50,12 +51,14 @@ type PrepareDataflowWriterOutput struct {
 type PrepareDataflowWriter struct {
 	Input  *PrepareDataflowWriterInput
 	Output *PrepareDataflowWriterOutput
+	DfA    dataflowaccessor.DataflowAccessor
+	SA     storageaccessor.StorageAccessor
 }
 
 // Launches the writer dataflow job.
 func (p *PrepareDataflowWriter) Transaction(ctx context.Context) error {
 	input := p.Input
-	writerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, input.TuningCfg)
+	writerTuningCfg, err := dataflowutils.UnmarshalDataflowTuningConfig(ctx, p.SA, input.TuningCfg)
 	if err != nil {
 		return fmt.Errorf("error reading writer tuning config %s: %v", input.TuningCfg, err)
 	}
@@ -82,11 +85,12 @@ func (p *PrepareDataflowWriter) Transaction(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	dfJobId, err := resource.CreateDataflowSMTResource(ctx, input.SmtJobId, dfLaunchReq)
+	dfJobId, err := resource.CreateDataflowSMTResource(ctx, p.DfA, input.SmtJobId, dfLaunchReq)
 	if err != nil {
 		return err
 	}
 	logger.Log.Info(fmt.Sprintf("Launched writer job with id: %s", dfJobId))
+	logger.Log.Info(fmt.Sprintf("\nEquivalent gCloud command for job %s:\n%s\n\n", dfLaunchReq.LaunchParameter.JobName, dataflowutils.GetGcloudDataflowCommand(dfLaunchReq)))
 	p.Output.JobId = dfJobId
 	return nil
 }
@@ -100,7 +104,7 @@ func validateUpdateWriterTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, s
 		cfg.ProjectId = spannerProjectId
 	}
 	if cfg.JobName == "" {
-		cfg.JobName = fmt.Sprintf("smt-writer-job-%s", utils.GenerateHashStr())
+		cfg.JobName = fmt.Sprintf("smt-reverse-replication-writer-%s", utils.GenerateHashStr())
 	}
 	if cfg.Location == "" {
 		cfg.Location = spannerLocation
@@ -114,7 +118,7 @@ func validateUpdateWriterTuningCfg(cfg *dataflowaccessor.DataflowTuningConfig, s
 	if cfg.MachineType == "" {
 		cfg.MachineType = "n1-standard-2"
 	}
-	cfg.AdditionalUserLabels["smt-writer-job"] = smtJobId
+	cfg.AdditionalUserLabels["smt-reverse-replication-writer"] = smtJobId
 	if cfg.GcsTemplatePath == "" {
 		cfg.GcsTemplatePath = constants.REVERSE_REPLICATION_WRITER_TEMPLATE_PATH
 	}
diff --git a/reverserepl/activity/prepare_gcs_bucket.go b/reverserepl/activity/prepare_gcs_bucket.go
index dc6538dd26..24db80398f 100644
--- a/reverserepl/activity/prepare_gcs_bucket.go
+++ b/reverserepl/activity/prepare_gcs_bucket.go
@@ -41,26 +41,27 @@ type PrepareGcsBucketOutput struct {
 
 type PrepareGcsBucket struct {
 	Input *PrepareGcsBucketInput
+	SA    storageaccessor.StorageAccessor
 }
 
 // This creates a GCS bucket if based on flag input. It subsequently uploads local files to the bucket.
 func (p *PrepareGcsBucket) Transaction(ctx context.Context) error {
 	input := p.Input
 	if input.IsSMTBucketRequired {
-		err := resource.CreateBucketSMTResource(ctx, input.SmtJobId, input.SmtBucketName, input.SpannerProjectId, input.SpannerLocation, nil, 45)
+		err := resource.CreateBucketSMTResource(ctx, p.SA, input.SmtJobId, input.SmtBucketName, input.SpannerProjectId, input.SpannerLocation, nil, 45)
 		if err != nil {
 			return err
 		}
 		logger.Log.Info(fmt.Sprintf("Created bucket: %s", input.SmtBucketName))
 		if !strings.HasPrefix(input.SessionFilePath, constants.GCS_FILE_PREFIX) {
-			err := storageaccessor.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "session.json", input.SessionFilePath)
+			err := p.SA.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "session.json", input.SessionFilePath)
 			if err != nil {
 				return fmt.Errorf("could not upload session file to GCS: %v", err)
 			}
 			logger.Log.Debug(fmt.Sprintf("Uploaded local session file: %s to bucket %s", input.SessionFilePath, input.SmtBucketName))
 		}
 		if !strings.HasPrefix(input.SourceConnectionConfig, constants.GCS_FILE_PREFIX) {
-			err := storageaccessor.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "source-connection-config.json", input.SourceConnectionConfig)
+			err := p.SA.UploadLocalFileToGCS(ctx, fmt.Sprintf("%s%s/", constants.GCS_FILE_PREFIX, input.SmtBucketName), "source-connection-config.json", input.SourceConnectionConfig)
 			if err != nil {
 				return fmt.Errorf("could not upload source connection config file to GCS: %v", err)
 			}
diff --git a/reverserepl/activity/prepare_metadata_db.go b/reverserepl/activity/prepare_metadata_db.go
index 052e11319d..f0a44b0377 100644
--- a/reverserepl/activity/prepare_metadata_db.go
+++ b/reverserepl/activity/prepare_metadata_db.go
@@ -35,12 +35,13 @@ type PrepareMetadataDbOutput struct {
 type PrepareMetadataDb struct {
 	Input  *PrepareMetadataDbInput
 	Output *PrepareMetadataDbOutput
+	SpA    spanneraccessor.SpannerAccessor
 }
 
 // Creates a metadata db for reverse replication if one is not already present.
 func (p *PrepareMetadataDb) Transaction(ctx context.Context) error {
 	input := p.Input
-	dbExists, err := spanneraccessor.CheckExistingDb(ctx, input.DbURI)
+	dbExists, err := p.SpA.CheckExistingDb(ctx, input.DbURI)
 	if err != nil {
 		return fmt.Errorf("error checking existing db: %v", err)
 	}
@@ -49,7 +50,7 @@ func (p *PrepareMetadataDb) Transaction(ctx context.Context) error {
 		p.Output.Exists = true
 		return nil
 	}
-	resource.CreateMetadataDbSMTResource(ctx, input.SmtJobId, input.DbURI)
+	resource.CreateMetadataDbSMTResource(ctx, p.SpA, input.SmtJobId, input.DbURI)
 	logger.Log.Info(fmt.Sprintf("Created reverse replication metadata db %s", input.DbURI))
 	p.Output.Created = true
 	return nil
diff --git a/reverserepl/create.go b/reverserepl/create.go
index 4e5249389a..1301745b33 100644
--- a/reverserepl/create.go
+++ b/reverserepl/create.go
@@ -17,7 +17,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"slices"
 	"strings"
 
 	spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner"
@@ -85,7 +84,7 @@ func validateAndUpdateJobData(ctx context.Context, request *JobData, uuid string
 	}
 	if request.FiltrationMode == "" {
 		request.FiltrationMode = constants.RR_READER_FILTER_FWD
-	} else if !slices.Contains([]string{constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE}, request.FiltrationMode) {
+	} else if !utils.Contains([]string{constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE}, request.FiltrationMode) {
 		return fmt.Errorf("found filtrationMode %s, only allowed values are [%s, %s]", request.FiltrationMode, constants.RR_READER_FILTER_FWD, constants.RR_READER_FILTER_NONE)
 	}
 	if request.TimerInterval < 1 {
@@ -95,6 +94,15 @@ func validateAndUpdateJobData(ctx context.Context, request *JobData, uuid string
 		request.WindowDuration = "10s"
 	}
 
+	if request.ShardingCustomJarPath != "" && request.ShardingCustomClassName == "" {
+		return fmt.Errorf("found non-empty value for ShardingCustomJarPath, but empty value for ShardingCustomClassName")
+	}
+	if request.ShardingCustomJarPath == "" && request.ShardingCustomClassName != "" {
+		return fmt.Errorf("found non-empty value for ShardingCustomClassName, but empty value for ShardingCustomJarPath")
+	}
+	if request.ShardingCustomJarPath != "" && !strings.HasPrefix(request.ShardingCustomJarPath, constants.GCS_FILE_PREFIX) {
+		return fmt.Errorf("please specify a valid GCS path for ShardingCustomJarPath, starting with gs://")
+	}
 	// Replace '-' with '_' since hyphens are not allowed in cs names.
 	request.ChangeStreamName = strings.Replace(request.ChangeStreamName, "-", "_", -1)
 
@@ -171,24 +179,26 @@ func CreateWorkflow(ctx context.Context, request JobData) error {
 		},
 		&rractivity.PrepareDataflowReader{
 			Input: &rractivity.PrepareDataflowReaderInput{
-				SmtJobId:             smtJobId,
-				ChangeStreamName:     request.ChangeStreamName,
-				InstanceId:           request.InstanceId,
-				DatabaseId:           request.DatabaseId,
-				SpannerProjectId:     request.SpannerProjectId,
-				SessionFilePath:      request.SessionFileGcsPath,
-				SourceShardsFilePath: request.SourceConnectionConfigGcsPath,
-				MetadataInstance:     request.MetadataInstance,
-				MetadataDatabase:     request.MetadataDatabase,
-				GcsOutputDirectory:   request.GcsDataDirectory,
-				StartTimestamp:       request.StartTimestamp,
-				EndTimestamp:         request.EndTimestamp,
-				WindowDuration:       request.WindowDuration,
-				FiltrationMode:       request.FiltrationMode,
-				MetadataTableSuffix:  request.MetadataTableSuffix,
-				SkipDirectoryName:    request.SkipDirectoryName,
-				TuningCfg:            request.ReaderCfg,
-				SpannerLocation:      request.SpannerLocation,
+				SmtJobId:                smtJobId,
+				ChangeStreamName:        request.ChangeStreamName,
+				InstanceId:              request.InstanceId,
+				DatabaseId:              request.DatabaseId,
+				SpannerProjectId:        request.SpannerProjectId,
+				SessionFilePath:         request.SessionFileGcsPath,
+				SourceShardsFilePath:    request.SourceConnectionConfigGcsPath,
+				MetadataInstance:        request.MetadataInstance,
+				MetadataDatabase:        request.MetadataDatabase,
+				GcsOutputDirectory:      request.GcsDataDirectory,
+				StartTimestamp:          request.StartTimestamp,
+				EndTimestamp:            request.EndTimestamp,
+				WindowDuration:          request.WindowDuration,
+				FiltrationMode:          request.FiltrationMode,
+				MetadataTableSuffix:     request.MetadataTableSuffix,
+				SkipDirectoryName:       request.SkipDirectoryName,
+				ShardingCustomJarPath:   request.ShardingCustomJarPath,
+				ShardingCustomClassName: request.ShardingCustomClassName,
+				TuningCfg:               request.ReaderCfg,
+				SpannerLocation:         request.SpannerLocation,
 			},
 			Output: &rractivity.PrepareDataflowReaderOutput{},
 		},
diff --git a/reverserepl/resource/resource.go b/reverserepl/resource/resource.go
index a7f342cdb6..79721fbf2b 100644
--- a/reverserepl/resource/resource.go
+++ b/reverserepl/resource/resource.go
@@ -27,21 +27,21 @@ import (
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/dao"
 )
 
-func CreateChangeStreamSMTResource(ctx context.Context, smtJobId, changeStreamName, dbURI string) error {
+func CreateChangeStreamSMTResource(ctx context.Context, spA spanneraccessor.SpannerAccessor, smtJobId, changeStreamName, dbURI string) error {
 	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
 	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_ChangeStream{DbURI: dbURI}}
 	err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, changeStreamName, changeStreamName, "change-stream", resourceData)
 	if err != nil {
 		return fmt.Errorf("error inserting SMT change stream resource: %v", err)
 	}
-	err = spanneraccessor.CreateChangeStream(ctx, changeStreamName, dbURI)
+	err = spA.CreateChangeStream(ctx, changeStreamName, dbURI)
 	if err != nil {
 		return fmt.Errorf("error in change stream creation: %v", err)
 	}
 	return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
 }
 
-func CreateMetadataDbSMTResource(ctx context.Context, smtJobId, dbURI string) error {
+func CreateMetadataDbSMTResource(ctx context.Context, spA spanneraccessor.SpannerAccessor, smtJobId, dbURI string) error {
 	_, _, dbName := utils.ParseDbURI(dbURI)
 	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
 	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_MetadataDb{DbURI: dbURI}}
@@ -49,14 +49,14 @@ func CreateMetadataDbSMTResource(ctx context.Context, smtJobId, dbURI string) er
 	if err != nil {
 		return fmt.Errorf("error inserting SMT metadata db resource: %v", err)
 	}
-	err = spanneraccessor.CreateEmptyDatabase(ctx, dbURI)
+	err = spA.CreateEmptyDatabase(ctx, dbURI)
 	if err != nil {
 		return fmt.Errorf("error creating db: %v", err)
 	}
 	return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
 }
 
-func CreateBucketSMTResource(ctx context.Context, smtJobId, bucketName, projectId, location string, matchesPrefix []string, ttl int64) error {
+func CreateBucketSMTResource(ctx context.Context, sa storageaccessor.StorageAccessor, smtJobId, bucketName, projectId, location string, matchesPrefix []string, ttl int64) error {
 	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
 	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_GCSBucket{
 		Name:          bucketName,
@@ -69,21 +69,21 @@ func CreateBucketSMTResource(ctx context.Context, smtJobId, bucketName, projectI
 	if err != nil {
 		return fmt.Errorf("error inserting SMT bucket resource: %v", err)
 	}
-	err = storageaccessor.CreateGCSBucketWithLifecycle(ctx, bucketName, projectId, location, matchesPrefix, ttl)
+	err = sa.CreateGCSBucketWithLifecycle(ctx, bucketName, projectId, location, matchesPrefix, ttl)
 	if err != nil {
 		return fmt.Errorf("error in bucket creation: %v", err)
 	}
 	return dao.UpdateSMTResourceState(ctx, resourceId, "CREATED")
 }
 
-func CreateDataflowSMTResource(ctx context.Context, smtJobId string, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (string, error) {
+func CreateDataflowSMTResource(ctx context.Context, da dataflowaccessor.DataflowAccessor, smtJobId string, launchRequest *dataflowpb.LaunchFlexTemplateRequest) (string, error) {
 	resourceId := fmt.Sprintf("smt-resource-%s", utils.GenerateHashStr())
 	resourceData := spanner.NullJSON{Valid: true, Value: ResourceData_Dataflow{LaunchRequest: launchRequest, EquivalentGcloudCmd: dataflowutils.GetGcloudDataflowCommand(launchRequest)}}
 	err := dao.InsertSMTResourceEntry(ctx, resourceId, smtJobId, "", launchRequest.LaunchParameter.JobName, "dataflow", resourceData)
 	if err != nil {
 		return "", fmt.Errorf("error inserting SMT dataflow resource: %v", err)
 	}
-	response, err := dataflowaccessor.LaunchDataflowJob(ctx, launchRequest)
+	response, err := da.LaunchFlexTemplate(ctx, launchRequest)
 	if err != nil {
 		return "", fmt.Errorf("error in launching dataflow job: %v", err)
 	}
diff --git a/reverserepl/types.go b/reverserepl/types.go
index ffc9fc283d..4884993724 100644
--- a/reverserepl/types.go
+++ b/reverserepl/types.go
@@ -21,22 +21,24 @@ type JobData struct {
 	SourceConnectionConfig string `json:"sourceConnectionConfig"`
 	SpannerProjectId       string `json:"spannerProjectId"`
 	// Optional parameters.
-	JobName                string `json:"jobName"`
-	SourceType             string `json:"sourceType"`
-	MetadataInstance       string `json:"metadataInstance"`
-	MetadataDatabase       string `json:"metadataDatabase"`
-	GcsDataDirectory       string `json:"gcsDataDirectory"`
-	ChangeStreamName       string `json:"changeStreamName"`
-	StartTimestamp         string `json:"startTimestamp"`
-	EndTimestamp           string `json:"endTimestamp"`
-	WindowDuration         string `json:"windowDuration"`
-	FiltrationMode         string `json:"filtrationMode"`
-	SourceDbTimezoneOffset string `json:"sourceDbTimezoneOffset"`
-	TimerInterval          int    `json:"timerInterval"`
-	MetadataTableSuffix    string `json:"metadataTableSuffix"`
-	SkipDirectoryName      string `json:"skipDirectoryName"`
-	ReaderCfg              string `json:"readerCfg"`
-	WriterCfg              string `json:"writerCfg"`
+	JobName                 string `json:"jobName"`
+	SourceType              string `json:"sourceType"`
+	MetadataInstance        string `json:"metadataInstance"`
+	MetadataDatabase        string `json:"metadataDatabase"`
+	GcsDataDirectory        string `json:"gcsDataDirectory"`
+	ChangeStreamName        string `json:"changeStreamName"`
+	StartTimestamp          string `json:"startTimestamp"`
+	EndTimestamp            string `json:"endTimestamp"`
+	WindowDuration          string `json:"windowDuration"`
+	FiltrationMode          string `json:"filtrationMode"`
+	SourceDbTimezoneOffset  string `json:"sourceDbTimezoneOffset"`
+	TimerInterval           int    `json:"timerInterval"`
+	MetadataTableSuffix     string `json:"metadataTableSuffix"`
+	SkipDirectoryName       string `json:"skipDirectoryName"`
+	ShardingCustomJarPath   string `json:"shardingCustomJarPath"`
+	ShardingCustomClassName string `json:"shardingCustomClassName"`
+	ReaderCfg               string `json:"readerCfg"`
+	WriterCfg               string `json:"writerCfg"`
 	// SMT generated - These fields are for use internally by SMT. These should not be configured when passing input.
 	IsSMTBucketRequired bool   `json:"isSMTBucketRequired"`
 	SmtBucketName       string `json:"smtBucketName"`