diff --git a/.github/workflows/deploy-dataflow.yaml b/.github/workflows/deploy-dataflow.yaml index bc2a8fd..67331df 100644 --- a/.github/workflows/deploy-dataflow.yaml +++ b/.github/workflows/deploy-dataflow.yaml @@ -72,7 +72,7 @@ jobs: JOB_NAME="${{ env.JOB_PREFIX }}-$(date +%Y%m%d-%H%M%S)" echo "Submitting ${{ inputs.environment }} job: $JOB_NAME" - python src/ml_pipelines_kfp/dataflow/iris_streaming_pipeline.py \ + python src/dataflow/iris_streaming_pipeline.py \ --input_topic ${{ env.PUBSUB_TOPIC }} \ --output_table ${{ env.OUTPUT_TABLE }} \ --project_id ${{ env.PROJECT_ID }} \ diff --git a/src/ml_pipelines_kfp/iris_xgboost/bq_dataloader.py b/scripts/bq_dataloader.py similarity index 100% rename from src/ml_pipelines_kfp/iris_xgboost/bq_dataloader.py rename to scripts/bq_dataloader.py diff --git a/src/ml_pipelines_kfp/iris_xgboost/load_data.sh b/scripts/load_data.sh similarity index 100% rename from src/ml_pipelines_kfp/iris_xgboost/load_data.sh rename to scripts/load_data.sh diff --git a/src/ml_pipelines_kfp/iris_xgboost/pubsub_producer.py b/scripts/pubsub_producer.py similarity index 100% rename from src/ml_pipelines_kfp/iris_xgboost/pubsub_producer.py rename to scripts/pubsub_producer.py diff --git a/src/ml_pipelines_kfp/dataflow/iris_streaming_pipeline.py b/src/dataflow/iris_streaming_pipeline.py similarity index 99% rename from src/ml_pipelines_kfp/dataflow/iris_streaming_pipeline.py rename to src/dataflow/iris_streaming_pipeline.py index 57943c8..563090d 100644 --- a/src/ml_pipelines_kfp/dataflow/iris_streaming_pipeline.py +++ b/src/dataflow/iris_streaming_pipeline.py @@ -175,6 +175,7 @@ def run_pipeline(argv=None): google_cloud_options = pipeline_options.view_as(GoogleCloudOptions) google_cloud_options.project = known_args.project_id + google_cloud_options.region = known_args.region with beam.Pipeline(options=pipeline_options) as pipeline: diff --git a/src/ml_pipelines_kfp/iris_xgboost/pipelines/test_inference.ipynb b/src/ml_pipelines_kfp/notebooks/test_inference.ipynb similarity index 100% rename from src/ml_pipelines_kfp/iris_xgboost/pipelines/test_inference.ipynb rename to src/ml_pipelines_kfp/notebooks/test_inference.ipynb diff --git a/test/test_pubsub.py b/test/test_pubsub.py index 1b2dde6..c8a72f5 100755 --- a/test/test_pubsub.py +++ b/test/test_pubsub.py @@ -18,7 +18,7 @@ PUBSUB_SUBSCRIPTION, PROJECT_ID ) -from ml_pipelines_kfp.iris_xgboost.pubsub_producer import IrisDataPubSubProducer +from scripts.pubsub_producer import IrisDataPubSubProducer def test_pubsub_connection():