From b9d4a049c3b7701af706fdfe5b6206f4f2ba7629 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:03:25 -0400 Subject: [PATCH 1/5] chore: removes old proof of concept --- .../services/centralized_service/__init__.py | 18 - .../services/centralized_service/_helpers.py | 24 -- .../services/centralized_service/client.py | 249 ------------- .../bigquery_v2/test_centralized_service.py | 332 ------------------ 4 files changed, 623 deletions(-) delete mode 100644 google/cloud/bigquery_v2/services/centralized_service/__init__.py delete mode 100644 google/cloud/bigquery_v2/services/centralized_service/_helpers.py delete mode 100644 google/cloud/bigquery_v2/services/centralized_service/client.py delete mode 100644 tests/unit/gapic/bigquery_v2/test_centralized_service.py diff --git a/google/cloud/bigquery_v2/services/centralized_service/__init__.py b/google/cloud/bigquery_v2/services/centralized_service/__init__.py deleted file mode 100644 index 03ddf6619..000000000 --- a/google/cloud/bigquery_v2/services/centralized_service/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .client import BigQueryClient - -__all__ = "BigQueryClient" diff --git a/google/cloud/bigquery_v2/services/centralized_service/_helpers.py b/google/cloud/bigquery_v2/services/centralized_service/_helpers.py deleted file mode 100644 index 53b585b18..000000000 --- a/google/cloud/bigquery_v2/services/centralized_service/_helpers.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -def _drop_self_key(kwargs): - "Drops 'self' key from a given kwargs dict." - - if not isinstance(kwargs, dict): - raise TypeError("kwargs must be a dict.") - kwargs.pop("self", None) # Essentially a no-op if 'self' key does not exist - return kwargs diff --git a/google/cloud/bigquery_v2/services/centralized_service/client.py b/google/cloud/bigquery_v2/services/centralized_service/client.py deleted file mode 100644 index 3b53fb53b..000000000 --- a/google/cloud/bigquery_v2/services/centralized_service/client.py +++ /dev/null @@ -1,249 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -from typing import ( - Optional, - Sequence, - Tuple, - Union, -) - -from google.cloud.bigquery_v2.services.centralized_service import _helpers - -# Import service client modules -from google.cloud.bigquery_v2.services import ( - dataset_service, - job_service, - model_service, -) - -# Import types modules (to access *Requests classes) -from google.cloud.bigquery_v2.types import ( - dataset, - job, - model, -) - -from google.api_core import client_options as client_options_lib -from google.api_core import gapic_v1 -from google.api_core import retry as retries -from google.auth import credentials as auth_credentials - -# Create a type alias -try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] -except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object, None] # type: ignore - -# TODO: This line is here to simplify prototyping, etc. -PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT") - -DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT -DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT -DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () - - -# Create Centralized Client -class BigQueryClient: - def __init__( - self, - *, - credentials: Optional[auth_credentials.Credentials] = None, - client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, - ): - self._clients = {} - self._credentials = credentials - self._client_options = client_options - - @property - def dataset_service_client(self): - if "dataset" not in self._clients: - from google.cloud.bigquery_v2.services import dataset_service - - self._clients["dataset"] = dataset_service.DatasetServiceClient( - credentials=self._credentials, client_options=self._client_options - ) - return self._clients["dataset"] - - @dataset_service_client.setter - def dataset_service_client(self, value): - # Check for the methods the centralized client exposes (to allow duck-typing) - required_methods = [ - "get_dataset", - "insert_dataset", - "patch_dataset", - "update_dataset", - "delete_dataset", - "list_datasets", - "undelete_dataset", - ] - for method in required_methods: - if not hasattr(value, method) or not callable(getattr(value, method)): - raise AttributeError( - f"Object assigned to dataset_service_client is missing a callable '{method}' method." - ) - self._clients["dataset"] = value - - @property - def job_service_client(self): - if "job" not in self._clients: - from google.cloud.bigquery_v2.services import job_service - - self._clients["job"] = job_service.JobServiceClient( - credentials=self._credentials, client_options=self._client_options - ) - return self._clients["job"] - - @job_service_client.setter - def job_service_client(self, value): - required_methods = [ - "get_job", - "insert_job", - "cancel_job", - "delete_job", - "list_jobs", - ] - for method in required_methods: - if not hasattr(value, method) or not callable(getattr(value, method)): - raise AttributeError( - f"Object assigned to job_service_client is missing a callable '{method}' method." - ) - self._clients["job"] = value - - @property - def model_service_client(self): - if "model" not in self._clients: - from google.cloud.bigquery_v2.services import model_service - - self._clients["model"] = model_service.ModelServiceClient( - credentials=self._credentials, client_options=self._client_options - ) - return self._clients["model"] - - @model_service_client.setter - def model_service_client(self, value): - required_methods = [ - "get_model", - "delete_model", - "patch_model", - "list_models", - ] - for method in required_methods: - if not hasattr(value, method) or not callable(getattr(value, method)): - raise AttributeError( - f"Object assigned to model_service_client is missing a callable '{method}' method." - ) - self._clients["model"] = value - - def get_dataset( - self, - request: Optional[Union[dataset.GetDatasetRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.dataset_service_client.get_dataset(**kwargs) - - def list_datasets( - self, - request: Optional[Union[dataset.ListDatasetsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.dataset_service_client.list_datasets(**kwargs) - - def list_jobs( - self, - request: Optional[Union[job.ListJobsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.job_service_client.list_jobs(**kwargs) - - def get_model( - self, - request: Optional[Union[model.GetModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.model_service_client.get_model(**kwargs) - - def delete_model( - self, - request: Optional[Union[model.DeleteModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - # The underlying GAPIC client returns None on success. - return self.model_service_client.delete_model(**kwargs) - - def patch_model( - self, - request: Optional[Union[model.PatchModelRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.model_service_client.patch_model(**kwargs) - - def list_models( - self, - request: Optional[Union[model.ListModelsRequest, dict]] = None, - *, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, - ): - """ - TODO: Docstring is purposefully blank. microgenerator will add automatically. - """ - kwargs = _helpers._drop_self_key(locals()) - return self.model_service_client.list_models(**kwargs) diff --git a/tests/unit/gapic/bigquery_v2/test_centralized_service.py b/tests/unit/gapic/bigquery_v2/test_centralized_service.py deleted file mode 100644 index 9160dc7b1..000000000 --- a/tests/unit/gapic/bigquery_v2/test_centralized_service.py +++ /dev/null @@ -1,332 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -from typing import ( - Optional, - Sequence, - Tuple, - Union, -) -from unittest import mock - -from google.api_core import client_options as client_options_lib -from google.api_core import gapic_v1 -from google.api_core import retry as retries -from google.auth import credentials as auth_credentials - -# --- IMPORT SERVICECLIENT MODULES --- -from google.cloud.bigquery_v2.services import ( - centralized_service, - dataset_service, - job_service, - model_service, -) - -# --- IMPORT TYPES MODULES (to access *Requests classes) --- -from google.cloud.bigquery_v2.types import ( - dataset, - job, - model, -) - -# --- TYPE ALIASES --- -try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] -except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object, None] # type: ignore - -AnyRequest = Union[ - dataset.GetDatasetRequest, - model.GetModelRequest, - model.DeleteModelRequest, - model.PatchModelRequest, - job.ListJobsRequest, - model.ListModelsRequest, -] - -# --- CONSTANTS --- -PROJECT_ID = "test-project" -DATASET_ID = "test_dataset" -JOB_ID = "test_job" -MODEL_ID = "test_model" -DEFAULT_ETAG = "test_etag" - -DEFAULT_RETRY: OptionalRetry = gapic_v1.method.DEFAULT -DEFAULT_TIMEOUT: Union[float, object] = gapic_v1.method.DEFAULT -DEFAULT_METADATA: Sequence[Tuple[str, Union[str, bytes]]] = () - -# --- HELPERS --- -def assert_client_called_once_with( - mock_method: mock.Mock, - request: AnyRequest, - retry: OptionalRetry = DEFAULT_RETRY, - timeout: Union[float, object] = DEFAULT_TIMEOUT, - metadata: Sequence[Tuple[str, Union[str, bytes]]] = DEFAULT_METADATA, -): - """Helper to assert a client method was called with default args.""" - mock_method.assert_called_once_with( - request=request, - retry=retry, - timeout=timeout, - metadata=metadata, - ) - - -# --- FIXTURES --- -@pytest.fixture -def mock_dataset_service_client(): - """Mocks the DatasetServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.dataset_service.DatasetServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -@pytest.fixture -def mock_job_service_client(): - """Mocks the JobServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.job_service.JobServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -@pytest.fixture -def mock_model_service_client(): - """Mocks the ModelServiceClient.""" - with mock.patch( - "google.cloud.bigquery_v2.services.model_service.ModelServiceClient", - autospec=True, - ) as mock_client: - yield mock_client - - -# TODO: figure out a solution for this... is there an easier way to feed in clients? -# TODO: is there an easier way to make mock_x_service_clients? -@pytest.fixture -def bq_client( - mock_dataset_service_client, mock_job_service_client, mock_model_service_client -): - """Provides a BigQueryClient with mocked underlying services.""" - client = centralized_service.BigQueryClient() - client.dataset_service_client = mock_dataset_service_client - client.job_service_client = mock_job_service_client - client.model_service_client = mock_model_service_client - ... - return client - - -# --- TEST CLASSES --- - -from google.api_core import client_options as client_options_lib - -# from google.api_core.client_options import ClientOptions -from google.auth import credentials as auth_credentials - -# from google.auth.credentials import Credentials - - -class TestCentralizedClientInitialization: - @pytest.mark.parametrize( - "credentials, client_options", - [ - (None, None), - (mock.MagicMock(spec=auth_credentials.Credentials), None), - ( - None, - client_options_lib.ClientOptions(api_endpoint="test.googleapis.com"), - ), - ( - mock.MagicMock(spec=auth_credentials.Credentials), - client_options_lib.ClientOptions(api_endpoint="test.googleapis.com"), - ), - ], - ) - def test_client_initialization_arguments( - self, - credentials, - client_options, - mock_dataset_service_client, - mock_job_service_client, - mock_model_service_client, - ): - # Act - client = centralized_service.BigQueryClient( - credentials=credentials, client_options=client_options - ) - - # Assert - # The BigQueryClient should have been initialized. Accessing the - # service client properties should instantiate them with the correct arguments. - - # Access the property to trigger instantiation - _ = client.dataset_service_client - mock_dataset_service_client.assert_called_once_with( - credentials=credentials, client_options=client_options - ) - - _ = client.job_service_client - mock_job_service_client.assert_called_once_with( - credentials=credentials, client_options=client_options - ) - - _ = client.model_service_client - mock_model_service_client.assert_called_once_with( - credentials=credentials, client_options=client_options - ) - - -class TestCentralizedClientDatasetService: - def test_get_dataset(self, bq_client, mock_dataset_service_client): - # Arrange - expected_dataset = dataset.Dataset( - kind="bigquery#dataset", id=f"{PROJECT_ID}:{DATASET_ID}" - ) - mock_dataset_service_client.get_dataset.return_value = expected_dataset - get_dataset_request = dataset.GetDatasetRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID - ) - - # Act - dataset_response = bq_client.get_dataset(request=get_dataset_request) - - # Assert - assert dataset_response == expected_dataset - assert_client_called_once_with( - mock_dataset_service_client.get_dataset, get_dataset_request - ) - - -class TestCentralizedClientJobService: - def test_list_jobs(self, bq_client, mock_job_service_client): - # Arrange - expected_jobs = [job.Job(kind="bigquery#job", id=f"{PROJECT_ID}:{JOB_ID}")] - mock_job_service_client.list_jobs.return_value = expected_jobs - list_jobs_request = job.ListJobsRequest(project_id=PROJECT_ID) - - # Act - jobs_response = bq_client.list_jobs(request=list_jobs_request) - - # Assert - assert jobs_response == expected_jobs - assert_client_called_once_with( - mock_job_service_client.list_jobs, list_jobs_request - ) - - -class TestCentralizedClientModelService: - def test_get_model(self, bq_client, mock_model_service_client): - # Arrange - expected_model = model.Model( - etag=DEFAULT_ETAG, - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - ) - mock_model_service_client.get_model.return_value = expected_model - get_model_request = model.GetModelRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID - ) - - # Act - model_response = bq_client.get_model(request=get_model_request) - - # Assert - assert model_response == expected_model - assert_client_called_once_with( - mock_model_service_client.get_model, get_model_request - ) - - def test_delete_model(self, bq_client, mock_model_service_client): - # Arrange - # The underlying service call returns nothing on success. - mock_model_service_client.delete_model.return_value = None - delete_model_request = model.DeleteModelRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID, model_id=MODEL_ID - ) - - # Act - # The wrapper method should also return nothing. - result = bq_client.delete_model(request=delete_model_request) - - # Assert - # 1. Assert the return value is None. This fails if the method doesn't exist. - assert result is None - # 2. Assert the underlying service was called correctly. - assert_client_called_once_with( - mock_model_service_client.delete_model, - delete_model_request, - ) - - def test_patch_model(self, bq_client, mock_model_service_client): - # Arrange - expected_model = model.Model( - etag="new_etag", - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - description="A newly patched description.", - ) - mock_model_service_client.patch_model.return_value = expected_model - - model_patch = model.Model(description="A newly patched description.") - patch_model_request = model.PatchModelRequest( - project_id=PROJECT_ID, - dataset_id=DATASET_ID, - model_id=MODEL_ID, - model=model_patch, - ) - - # Act - patched_model = bq_client.patch_model(request=patch_model_request) - - # Assert - assert patched_model == expected_model - assert_client_called_once_with( - mock_model_service_client.patch_model, patch_model_request - ) - - def test_list_models(self, bq_client, mock_model_service_client): - # Arrange - expected_models = [ - model.Model( - etag=DEFAULT_ETAG, - model_reference={ - "project_id": PROJECT_ID, - "dataset_id": DATASET_ID, - "model_id": MODEL_ID, - }, - ) - ] - mock_model_service_client.list_models.return_value = expected_models - list_models_request = model.ListModelsRequest( - project_id=PROJECT_ID, dataset_id=DATASET_ID - ) - # Act - models_response = bq_client.list_models(request=list_models_request) - - # Assert - assert models_response == expected_models - assert_client_called_once_with( - mock_model_service_client.list_models, list_models_request - ) From 5b4d538a1053c5381c9853e9337a58f1ecc69e56 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:10:11 -0400 Subject: [PATCH 2/5] removes old __init__.py --- google/cloud/bigquery_v2/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/google/cloud/bigquery_v2/__init__.py b/google/cloud/bigquery_v2/__init__.py index 30b6a63d6..83c82e729 100644 --- a/google/cloud/bigquery_v2/__init__.py +++ b/google/cloud/bigquery_v2/__init__.py @@ -24,7 +24,6 @@ from .services.routine_service import RoutineServiceClient from .services.row_access_policy_service import RowAccessPolicyServiceClient from .services.table_service import TableServiceClient -from .services.centralized_service import BigQueryClient from .types.biglake_config import BigLakeConfiguration from .types.clustering import Clustering @@ -215,7 +214,6 @@ "BiEngineReason", "BiEngineStatistics", "BigLakeConfiguration", - "BigQueryClient", "BigtableColumn", "BigtableColumnFamily", "BigtableOptions", From 132c571224b93f0220088ad3f2d641490775c694 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:28:50 -0400 Subject: [PATCH 3/5] Adds two utility files to handle basic tasks --- scripts/microgenerator/name_utils.py | 73 ++++++++++++++++ scripts/microgenerator/utils.py | 120 +++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 scripts/microgenerator/name_utils.py create mode 100644 scripts/microgenerator/utils.py diff --git a/scripts/microgenerator/name_utils.py b/scripts/microgenerator/name_utils.py new file mode 100644 index 000000000..129050c37 --- /dev/null +++ b/scripts/microgenerator/name_utils.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility module for handling name transformations.""" + +import re +from typing import Dict + + +def to_snake_case(name: str) -> str: + """Converts a PascalCase name to snake_case.""" + return re.sub(r"(? Dict[str, str]: + """ + Generates various name formats for a service based on its client class name. + + Args: + class_name: The PascalCase name of the service client class + (e.g., 'DatasetServiceClient'). + + Returns: + A dictionary containing different name variations. + """ + snake_case_name = to_snake_case(class_name) + module_name = snake_case_name.replace("_client", "") + service_name = module_name.replace("_service", "") + + return { + "service_name": service_name, + "service_module_name": module_name, + "service_client_class": class_name, + "property_name": snake_case_name, # Direct use of snake_case_name + } + + +def method_to_request_class_name(method_name: str) -> str: + """ + Converts a snake_case method name to a PascalCase Request class name. + + This follows the convention where a method like `get_dataset` corresponds + to a `GetDatasetRequest` class. + + Args: + method_name: The snake_case name of the API method. + + Returns: + The inferred PascalCase name for the corresponding request class. + + Example: + >>> method_to_request_class_name('get_dataset') + 'GetDatasetRequest' + >>> method_to_request_class_name('list_jobs') + 'ListJobsRequest' + """ + # e.g., "get_dataset" -> ["get", "dataset"] + parts = method_name.split("_") + # e.g., ["get", "dataset"] -> "GetDataset" + pascal_case_base = "".join(part.capitalize() for part in parts) + return f"{pascal_case_base}Request" diff --git a/scripts/microgenerator/utils.py b/scripts/microgenerator/utils.py new file mode 100644 index 000000000..c81387d57 --- /dev/null +++ b/scripts/microgenerator/utils.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utility functions for the microgenerator.""" + +import os +import sys +import yaml +import jinja2 +from typing import Dict, Any, Iterator, Callable + + +def _load_resource( + loader_func: Callable, + path: str, + not_found_exc: type, + parse_exc: type, + resource_type_name: str, +) -> Any: + """ + Generic resource loader with common error handling. + + Args: + loader_func: A callable that performs the loading and returns the resource. + It should raise appropriate exceptions on failure. + path: The path/name of the resource for use in error messages. + not_found_exc: The exception type to catch for a missing resource. + parse_exc: The exception type to catch for a malformed resource. + resource_type_name: A human-readable name for the resource type. + """ + try: + return loader_func() + except not_found_exc: + print(f"Error: {resource_type_name} '{path}' not found.", file=sys.stderr) + sys.exit(1) + except parse_exc as e: + print( + f"Error: Could not load {resource_type_name.lower()} from '{path}': {e}", + file=sys.stderr, + ) + sys.exit(1) + + +def load_template(template_path: str) -> jinja2.Template: + """ + Loads a Jinja2 template from a given file path. + """ + template_dir = os.path.dirname(template_path) + template_name = os.path.basename(template_path) + + def _loader() -> jinja2.Template: + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir or "."), + trim_blocks=True, + lstrip_blocks=True, + ) + return env.get_template(template_name) + + return _load_resource( + loader_func=_loader, + path=template_path, + not_found_exc=jinja2.exceptions.TemplateNotFound, + parse_exc=jinja2.exceptions.TemplateError, + resource_type_name="Template file", + ) + + +def load_config(config_path: str) -> Dict[str, Any]: + """Loads the generator's configuration from a YAML file.""" + + def _loader() -> Dict[str, Any]: + with open(config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + return _load_resource( + loader_func=_loader, + path=config_path, + not_found_exc=FileNotFoundError, + parse_exc=yaml.YAMLError, + resource_type_name="Configuration file", + ) + + +def walk_codebase(path: str) -> Iterator[str]: + """Yields all .py file paths in a directory.""" + for root, _, files in os.walk(path): + for file in files: + if file.endswith(".py"): + yield os.path.join(root, file) + + +def write_code_to_file(output_path: str, content: str): + """Ensures the output directory exists and writes content to the file.""" + output_dir = os.path.dirname(output_path) + + # An empty output_dir means the file is in the current directory. + if output_dir: + print(f" Ensuring output directory exists: {os.path.abspath(output_dir)}") + os.makedirs(output_dir, exist_ok=True) + if not os.path.isdir(output_dir): + print(f" Error: Output directory was not created.", file=sys.stderr) + sys.exit(1) + + print(f" Writing generated code to: {os.path.abspath(output_path)}") + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) + print(f"Successfully generated {output_path}") From 90b224eef468a856c49edff308d65f6c2cd5a997 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:54:36 -0400 Subject: [PATCH 4/5] Adds a configuration file for the microgenerator --- scripts/microgenerator/config.yaml | 75 ++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scripts/microgenerator/config.yaml diff --git a/scripts/microgenerator/config.yaml b/scripts/microgenerator/config.yaml new file mode 100644 index 000000000..a9c608f29 --- /dev/null +++ b/scripts/microgenerator/config.yaml @@ -0,0 +1,75 @@ +# config.yaml + +# The name of the service, used for variable names and comments. +service_name: "bigquery" + +# A list of paths to the source code files to be parsed. +# Globs are supported. +source_files: + services: + - "google/cloud/bigquery_v2/services/dataset_service/client.py" + - "google/cloud/bigquery_v2/services/job_service/client.py" + - "google/cloud/bigquery_v2/services/model_service/client.py" + - "google/cloud/bigquery_v2/services/project_service/client.py" + - "google/cloud/bigquery_v2/services/routine_service/client.py" + - "google/cloud/bigquery_v2/services/row_access_policy_service/client.py" + - "google/cloud/bigquery_v2/services/table_service/client.py" + types: + - "google/cloud/bigquery_v2/types/dataset.py" + - "google/cloud/bigquery_v2/types/job.py" + - "google/cloud/bigquery_v2/types/model.py" + - "google/cloud/bigquery_v2/types/project.py" + - "google/cloud/bigquery_v2/types/routine.py" + - "google/cloud/bigquery_v2/types/row_access_policy.py" + - "google/cloud/bigquery_v2/types/table.py" + + +# Filtering rules for classes and methods. +filter: + classes: + # Only include classes with these suffixes. + include_suffixes: + - "ServiceClient" + - "Request" + # Exclude classes with these suffixes. + exclude_suffixes: + - "BigQueryClient" + methods: + # Include methods with these prefixes. + include_prefixes: + - "batch_delete_" + - "cancel_" + - "create_" + - "delete_" + - "get_" + - "insert_" + - "list_" + - "patch_" + - "undelete_" + - "update_" + # Exclude methods with these prefixes. + exclude_prefixes: + - "get_mtls_endpoint_and_cert_source" + overrides: + patch_table: + request_class_name: "UpdateOrPatchTableRequest" + patch_dataset: + request_class_name: "UpdateOrPatchDatasetRequest" + +# A list of templates to render and their corresponding output files. +# A list of templates to render and their corresponding output files. +templates: + - template: "templates/client.py.j2" + output: "google/cloud/bigquery_v2/services/centralized_service/client.py" + - template: "templates/_helpers.py.j2" + output: "google/cloud/bigquery_v2/services/centralized_service/_helpers.py" + - template: "templates/__init__.py.j2" + output: "google/cloud/bigquery_v2/services/centralized_service/__init__.py" + +post_processing_templates: + - template: "templates/post-processing/init.py.j2" + target_file: "google/cloud/bigquery_v2/__init__.py" + add_imports: + - "from .services.centralized_service import BigQueryClient" + add_to_all: + - "BigQueryClient" \ No newline at end of file From e071eabdc8c33c680755c5463fc2d5e9ee78adf6 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Sep 2025 12:58:05 -0400 Subject: [PATCH 5/5] Removes unused comment --- scripts/microgenerator/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/microgenerator/config.yaml b/scripts/microgenerator/config.yaml index a9c608f29..57330af31 100644 --- a/scripts/microgenerator/config.yaml +++ b/scripts/microgenerator/config.yaml @@ -56,7 +56,6 @@ filter: patch_dataset: request_class_name: "UpdateOrPatchDatasetRequest" -# A list of templates to render and their corresponding output files. # A list of templates to render and their corresponding output files. templates: - template: "templates/client.py.j2"