Added unified datacommons CLI tool

.github/workflows/ci.yaml

@@ -26,7 +26,7 @@ jobs:
         run: uv sync --locked
 
       - name: Check formatting with Ruff
-        run: uv run ruff format --check
+        run: uv run --extra lint ruff format --check
 
       # TODO(dwnoble): Fix formatting issues in datacommons-schema and uncomment these
       #- name: Lint with Ruff
@@ -35,4 +35,4 @@ jobs:
       - name: Run tests
         run: |
           # Enable parallel test execution
-          uv run pytest
+          uv run --extra test pytest

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     hooks:
       - id: run-tests
         name: Run tests
-        entry: uv run pytest
+        entry: uv run --extra test pytest
         language: system
         pass_filenames: false
         stages: [pre-push]

README.md

@@ -29,16 +29,17 @@ This section will guide you through setting up Data Commons locally and defining
 
 To get started, you'll need to check out the Data Commons repository and set up your local environment.
 
-#### Check out the repository:
+#### Clone the repository:
 
 ```bash
 git clone https://github.org/datacommonsorg/datacommons
 cd datacommons
 ```
 
-The repository contains three main components:
+The repository contains four main components:
 - `datacommons-api`: The REST API server for interacting with Data Commons
 - `datacommons-db`: The database layer for storing and querying data
+- `datacommons-cli`: Command-line interface for interacting with Data Commons
 - `datacommons-schema`: Schema management and validation tools
 
 #### Create a virtual environment with uv
@@ -52,7 +53,7 @@ uv sync
 Run the test suite to verify your setup:
 
 ```bash
-uv run pytest
+uv run --extra test pytest
 ```
 
 Tests are also run automatically before pushing changes.
@@ -71,14 +72,23 @@ Replace the values with your actual GCP project and Spanner instance details. Yo
 
 #### Start Data Commons:
 
-Run the `datacommons-api` command using `uv` to start a local development server.
+Run the `datacommons` command using `uv` to start a local development server.
 
 ```bash
-uv run datacommons-api
+uv run datacommons api start
 ```
 
 This will start the Data Commons API server on port 5000, ready to receive your schema and data.
 
+Alternatively, you can set the spanner configuration using command line arguments, which will take precedence over environment variables:
+
+```bash
+uv run datacommons api start \
+  --gcp-project-id="your-gcp-project-id" \
+  --gcp-spanner-instance-id="your-spanner-instance-id" \
+  --gcp-spanner-database-name="your-spanner-database-name"
+```
+
 ### 2. Define Your Schema
 
 Data Commons uses JSON-LD to define schemas, building upon RDF, RDFS, and SHACL for robust data modeling and validation. The repository includes example schema and data files in the `examples` directory:
@@ -316,4 +326,19 @@ You should see the:
     }
   ]
 }
-```
+```
+
+### Schema Tools
+
+Use the `datacommons schema` command to convert between MCF and JSON-LD formats.
+
+```bash
+# Convert with default settings
+uv run datacommons schema mcf2jsonld data.mcf
+
+# Convert with custom namespace and output file
+uv run datacommons schema mcf2jsonld data.mcf -n "dc:https://datacommons.org/" -o output.jsonld
+
+# Generate compact output
+uv run datacommons schema mcf2jsonld data.mcf -c
+```

packages/datacommons-api/datacommons_api/app_cli.py → packages/datacommons-api/datacommons_api/api_cli.py

@@ -16,26 +16,50 @@
 import uvicorn
 
 from datacommons_api.app import app
-from datacommons_api.core.config import get_config
+from datacommons_api.core.config import initialize_config
 from datacommons_api.core.logging import get_logger, setup_logging
 from datacommons_db.session import initialize_db
 
-
 setup_logging()
 logger = get_logger(__name__)
 
 
-@click.command()
+@click.group()
+def api():
+    """Data Commons API CLI suite"""
+    pass
+
+
+@api.command()
 @click.option("--host", default="127.0.0.1", help="Host to bind to.")
 @click.option("--port", default=5000, help="Port to listen on.")
 @click.option("--reload", is_flag=True, help="Enable auto-reload.")
-def main(host: str, port: int, *, reload: bool = False):
+@click.option("--gcp-project-id", default="", help="GCP project id.")
+@click.option("--gcp-spanner-instance-id", default="", help="GCP Spanner instance id.")
+@click.option(
+    "--gcp-spanner-database-name", default="", help="GCP Spanner database name."
+)
+def start(
+    host: str,
+    port: int,
+    reload: bool,
+    gcp_project_id: str,
+    gcp_spanner_instance_id: str,
+    gcp_spanner_database_name: str,
+):
     """Start the FastAPI app with Uvicorn."""
     logger.info("Starting Data Commons...")
-    config = get_config()
+    config = initialize_config(
+        gcp_project_id=gcp_project_id,
+        gcp_spanner_instance_id=gcp_spanner_instance_id,
+        gcp_spanner_database_name=gcp_spanner_database_name,
+    )
 
     # Initialize the database
     logger.info("Initializing database...")
+    logger.info("GCP Project ID: %s", config.GCP_PROJECT_ID)
+    logger.info("GCP Spanner Instance ID: %s", config.GCP_SPANNER_INSTANCE_ID)
+    logger.info("GCP Spanner Database Name: %s", config.GCP_SPANNER_DATABASE_NAME)
     initialize_db(
         config.GCP_PROJECT_ID,
         config.GCP_SPANNER_INSTANCE_ID,

packages/datacommons-api/datacommons_api/core/config.py

@@ -56,6 +56,10 @@ class ProductionConfig(Config):
 }
 
 
+# Default configuration
+app_config = config[os.getenv("APP_ENV", "default")]()
+
+
 def validate_config_or_exit(config: Config) -> None:
     """Ensure the configuration is valid"""
     # Ensure GCP Spanner is configured
@@ -65,9 +69,38 @@ def validate_config_or_exit(config: Config) -> None:
             sys.exit(1)
 
 
-def get_config() -> Config:
-    """Get the appropriate configuration object based on environment."""
-    env = os.getenv("APP_ENV", "default")
-    app_config = config[env]()
+def initialize_config(
+    gcp_project_id: str = "",
+    gcp_spanner_instance_id: str = "",
+    gcp_spanner_database_name: str = "",
+) -> Config:
+    """
+    Initialize the configuration object based on environment or command line arguments.
+
+    Args:
+        gcp_project_id: Optional GCP project id.
+        gcp_spanner_instance_id: Optional GCP Spanner instance id.
+        gcp_spanner_database_name: Optional GCP Spanner database name.
+
+    Returns:
+        Config: The configuration object.
+    """
+    app_config.GCP_PROJECT_ID = gcp_project_id or app_config.GCP_PROJECT_ID
+    app_config.GCP_SPANNER_INSTANCE_ID = (
+        gcp_spanner_instance_id or app_config.GCP_SPANNER_INSTANCE_ID
+    )
+    app_config.GCP_SPANNER_DATABASE_NAME = (
+        gcp_spanner_database_name or app_config.GCP_SPANNER_DATABASE_NAME
+    )
     validate_config_or_exit(app_config)
     return app_config
+
+
+def get_config() -> Config:
+    """
+    Get the configuration object.
+
+    Returns:
+        Config: The configuration object.
+    """
+    return app_config

packages/datacommons-api/pyproject.toml

@@ -12,14 +12,12 @@ dependencies = [
   "uvicorn[standard]>=0.22.0",
   "sqlalchemy>=2.0.0",
   "pydantic>=2.0.0",
+  "click>=8.1.7",
   "datacommons-db",
   "datacommons-schema",
 ]
 urls = {Homepage = "https://github.com/datacommonsorg/datacommons"}
 
-[project.scripts]
-datacommons-api = "datacommons_api.app_cli:main"
-
 [build-system]
 requires = ["uv", "setuptools"]
 build-backend = "setuptools.build_meta"

packages/datacommons-cli/datacommons_cli/cli.py

@@ -0,0 +1,33 @@
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from datacommons_api.core.logging import get_logger, setup_logging
+from datacommons_api.api_cli import api as api_cli
+from datacommons_schema.schema_cli import schema as schema_cli
+
+setup_logging()
+logger = get_logger(__name__)
+
+
+@click.group()
+def cli():
+    """Datacommons CLI suite"""
+    pass
+
+
+# Add schema CLI commands to the main CLI
+cli.add_command(api_cli)
+cli.add_command(schema_cli)

packages/datacommons-cli/datacommons_cli/version.py

@@ -0,0 +1,15 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.0.1"

packages/datacommons-cli/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "datacommons-cli"
+dynamic = ["version"]
+description = 'Data Commons CLI'
+license = "Apache-2.0"
+readme = "README.md"
+
+requires-python = ">=3.11"
+keywords = []
+authors = []
+dependencies = [
+  "click>=8.1.7",
+  "datacommons-api",
+  "datacommons-db",
+  "datacommons-schema",
+]
+urls = {Homepage = "https://github.com/datacommonsorg/datacommons"}
+
+[project.scripts]
+datacommons = "datacommons_cli.cli:cli"
+
+[build-system]
+requires = ["uv", "setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.dynamic]
+version = {attr = "datacommons_cli.version.__version__"}

packages/datacommons-schema/README.md

@@ -17,13 +17,13 @@ The `mcf2jsonld` command converts MCF files to JSON-LD format, with support for
 
 ```bash
 # Basic usage
-datacommons mcf2jsonld input.mcf
+datacommons schema mcf2jsonld input.mcf
 
 # With custom namespace
-datacommons mcf2jsonld input.mcf --namespace "schema:https://schema.org/"
+datacommons schema mcf2jsonld input.mcf --namespace "schema:https://schema.org/"
 
 # Output to file with compact format
-datacommons mcf2jsonld input.mcf -o output.jsonld -c
+datacommons schema mcf2jsonld input.mcf -o output.jsonld -c
 ```
 
 #### Options
@@ -70,25 +70,6 @@ mcf_nodes = parse_mcf_string(mcf_content)
 jsonld = mcf_nodes_to_jsonld(mcf_nodes, compact=True)
 ```
 
-### Command Line
-
-```bash
-# Convert with default settings
-datacommons mcf2jsonld data.mcf
-
-# Convert with custom namespace and output file
-datacommons mcf2jsonld data.mcf -n "dc:https://datacommons.org/" -o output.jsonld
-
-# Generate compact output
-datacommons mcf2jsonld data.mcf -c
-```
-
-## Dependencies
-
-- Click (for CLI interface)
-- Pydantic (for data validation)
-- JSON-LD processing libraries
-
 ## Contributing
 
 When contributing to this module:

packages/datacommons-schema/datacommons_schema/schema_cli.py

@@ -7,11 +7,11 @@
 
 
 @click.group()
-def cli():
+def schema():
     """Data Commons Schema Parsing CLI"""
 
 
-@cli.command()
+@schema.command()
 @click.argument("mcf_file", type=click.Path(exists=True))
 @click.option(
     "--namespace",

packages/datacommons-schema/pyproject.toml

@@ -18,16 +18,20 @@ classifiers = [
 dependencies = [
   "click",
   "pydantic",
-  "pytest"
+  "pyshacl",
+  "rdflib",
+]
+
+[project.optional-dependencies]
+test = [
+  "pytest",
 ]
 
 [project.urls]
 Documentation = "https://github.com/datacommonsorg/datacommons#readme"
 Issues = "https://github.com/datacommonsorg/datacommons/issues"
 Source = "https://github.com/datacommonsorg/datacommons"
 
-[project.scripts]
-datacommons-schema = "datacommons_schema.schema_cli:cli"
 
 [build-system]
 requires = ["uv", "setuptools"]