Add system test resource cleanup script to Google provider

docs/spelling_wordlist.txt

@@ -1549,6 +1549,7 @@ Splunk
 Sql
 sql
 sqla
+sqladmin
 Sqlalchemy
 sqlalchemy
 sqlglot

providers/google/tests/system/google/resources_cleanup/.airflowignore

@@ -0,0 +1 @@
+*

providers/google/tests/system/google/resources_cleanup/.gitignore

@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ress.egg-info/
+*.egg-info/
+*.iml
+__pycache__/
+build/
+dist/
+output/
+resources/

providers/google/tests/system/google/resources_cleanup/README.md

@@ -0,0 +1,110 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Google system test resource cleanup
+
+This helper project manages resources in a GCP project used for Google provider
+system tests. It lives under `providers/google/tests` as test tooling and is not
+included in the Google provider package.
+
+Create and activate a virtual environment, then install this helper from this
+directory:
+
+```shell
+pip install -e .
+```
+
+Now you can use `airflow-google-system-test-cleanup --help` to see the available
+commands.
+
+Here is a sample output:
+
+```shell
+usage: airflow-google-system-test-cleanup [-h] [--config-path CONFIG_PATH] [--resources-file-path RESOURCES_FILE_PATH] {list,list-asset-types,tree,delete} ...
+
+CLI to manage resource for a GCP project
+
+positional arguments:
+  {list,list-asset-types,tree,delete}
+    list                Retrieve the GCP resources for the given GCP project
+    list-asset-types    List all the unique asset types hierarchically in the GCP project
+    tree                Show the resources hierarchically as an HTML file
+    delete              Delete the resources for the given GCP project
+
+options:
+  -h, --help            show this help message and exit
+  --config-path CONFIG_PATH
+                        Direct path to a project config JSON file
+  --resources-file-path RESOURCES_FILE_PATH
+                        Direct path to the resources.json file
+```
+
+## Global Options
+
+-   `--config-path`: Override the automatic lookup of the project configuration.
+    Defaults to `config/<PROJECT_ID>.json`.
+-   `--resources-file-path`: Override where the tool saves or loads resource
+    data. Defaults to `resources/<PROJECT_ID>/resources.json`.
+
+## Example usages
+
+-   To retrieve all resources for the project and sync with Cloud Asset
+    Inventory: `airflow-google-system-test-cleanup list --project-id <PROJECT_ID> --sync`
+
+
+-   To list all resources from an existing previously synced file:
+    `airflow-google-system-test-cleanup list --project-id <PROJECT_ID>`
+
+
+-   To retrieve the specific asset type (e.g: `ai`) resources for the project:
+    `airflow-google-system-test-cleanup list --project-id <PROJECT_ID> --asset-type <ASSET_TYPE>`
+
+-   To produce an HTML tree visualization using a specific config file:
+    `airflow-google-system-test-cleanup tree --project-id <PROJECT_ID> --config-path /path/to/my_config.json`
+
+-   To list the all unique asset types in a hierarchical tree:
+    `airflow-google-system-test-cleanup list-asset-types --project-id <PROJECT_ID>`
+
+    > # you can pass `--asset-type` parameter to list only one type of assets.
+
+-   To clean up resources: `airflow-google-system-test-cleanup delete --project-id <PROJECT_ID>`
+
+-   To clean up only resources that are old enough:
+    `airflow-google-system-test-cleanup delete --project-id <PROJECT_ID> --asset-type dataproc --min-age-days 3`
+
+-   To skip a service group during deletion, for example Composer:
+    `airflow-google-system-test-cleanup delete --project-id <PROJECT_ID> --skip-asset-type composer`
+
+## Development & Testing
+
+To install the development dependencies and run tests:
+
+```shell
+pip install -e ".[test]"
+pytest
+```
+
+## Building and Distribution
+
+To build the project as a Python wheel for distribution:
+
+1. Install the build tool: `pip install build`
+2. Generate the wheel package: `python -m build`
+
+This will create the `dist/` directory containing the wheel file.

providers/google/tests/system/google/resources_cleanup/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

providers/google/tests/system/google/resources_cleanup/airflow_google_provider_resource_cleanup/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

providers/google/tests/system/google/resources_cleanup/airflow_google_provider_resource_cleanup/__main__.py

@@ -0,0 +1,132 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import argparse
+import asyncio
+import time
+from collections.abc import Callable
+
+from airflow_google_provider_resource_cleanup import constants as c
+from airflow_google_provider_resource_cleanup.commands.cmd_delete import handle_delete
+from airflow_google_provider_resource_cleanup.commands.cmd_list import handle_list
+from airflow_google_provider_resource_cleanup.commands.cmd_list_asset_types import handle_list_asset_types
+from airflow_google_provider_resource_cleanup.commands.cmd_tree import handler_tree
+
+HANDLERS = {
+    "list": handle_list,
+    "list-asset-types": handle_list_asset_types,
+    "tree": handler_tree,
+    "delete": handle_delete,
+}
+
+
+def _init_argparse() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="airflow-google-system-test-cleanup",
+        description="CLI to manage resources for a GCP project",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # Global options
+    parser.add_argument("--config-path", help="Direct path to a project config JSON file")
+    parser.add_argument("--resources-file-path", help="Direct path to the resources.json file")
+
+    # command: list
+    parser_list = subparsers.add_parser(
+        "list",
+        help="Retrieve the GCP resources for the given GCP project",
+    )
+    parser_list.add_argument("--project-id", help="", required=True)
+    parser_list.add_argument("--asset-type", help="", choices=c.ASSET_TYPE_OPTIONS)
+    parser_list.add_argument(
+        "--sync",
+        action="store_true",
+        default=False,
+    )
+
+    # command: list-asset-types
+    parser_list_asset_types = subparsers.add_parser(
+        "list-asset-types",
+        help="List all the unique asset types hierarchically in the GCP project",
+    )
+    parser_list_asset_types.add_argument("--project-id", help="", required=True)
+    parser_list_asset_types.add_argument("--asset-type", help="", choices=c.ASSET_TYPE_OPTIONS)
+
+    # command: tree
+    parser_tree = subparsers.add_parser(
+        "tree",
+        help="Show the resources hierarchically as an HTML file",
+    )
+    parser_tree.add_argument("--project-id", help="", required=True)
+    parser_tree.add_argument("--asset-type", help="", choices=c.ASSET_TYPE_OPTIONS)
+
+    # command: cleanup
+    parser_delete = subparsers.add_parser(
+        "delete",
+        help="Delete the resources for the given GCP project",
+    )
+    parser_delete.add_argument("--project-id", help="", required=True)
+    parser_delete.add_argument("--asset-type", help="", choices=c.ASSET_TYPE_OPTIONS)
+
+    parser_delete.add_argument(
+        "--min-age-days",
+        type=int,
+        help="Delete only resources created at least this many days ago",
+    )
+    parser_delete.add_argument(
+        "--skip-asset-type",
+        action="append",
+        default=[],
+        choices=c.ASSET_TYPE_OPTIONS,
+        help="Asset type group to skip during deletion. Can be used multiple times.",
+    )
+
+    return parser
+
+
+def main():
+    _parser = _init_argparse()
+    _args: argparse.Namespace = _parser.parse_args()
+    handler: Callable[[argparse.ArgumentParser, argparse.Namespace], None] = HANDLERS[_args.command]
+    start_time = time.monotonic()
+
+    try:
+        if asyncio.iscoroutinefunction(handler):
+            asyncio.run(handler(_parser, _args))
+        else:
+            handler(_parser, _args)
+    except Exception as e:
+        print("-" * 10, "EXCEPTION DURING SCRIPT EXECUTION", "-" * 10)
+        print(e)
+        print("-" * 40)
+
+    end_time = time.monotonic()
+    duration = end_time - start_time
+    hours, rem = divmod(duration, 3600)
+    minutes, seconds = divmod(rem, 60)
+    parts = []
+    if hours > 1:
+        parts.append(f"{int(hours)} hours")
+    if minutes > 1:
+        parts.append(f"{int(minutes)} minutes")
+    parts.append(f"{seconds:.2f} seconds")
+    print("Script duration after parsing: ", " ".join(parts))
+
+
+if __name__ == "__main__":
+    main()

providers/google/tests/system/google/resources_cleanup/airflow_google_provider_resource_cleanup/commands/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.