From ad95978378abf78a111648716a877603bf92f211 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 23 Oct 2025 12:08:31 +0200 Subject: [PATCH 1/6] Run all steps on @nvct --- examples/image_classifier/image_classifier_flow.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/image_classifier/image_classifier_flow.py b/examples/image_classifier/image_classifier_flow.py index 96b4fbb..56d6797 100644 --- a/examples/image_classifier/image_classifier_flow.py +++ b/examples/image_classifier/image_classifier_flow.py @@ -35,7 +35,8 @@ class ImageClassifierFlow(FlowSpec): @pypi(python="3.11.9", packages={"torchvision": "0.19.1"}) @card(type="default") - @kubernetes + #@kubernetes + @nvct @step def start(self): import torchvision @@ -155,7 +156,8 @@ def train(self): }, ) # Check https://docs.metaflow.org/api/step-decorators/kubernetes for details on @kubernetes decorator - @kubernetes(cpu=1, memory=4096) + #@kubernetes(cpu=1, memory=4096) + @nvct @step def evaluate(self): import torch @@ -196,7 +198,8 @@ def evaluate(self): self.next(self.upload_model_to_gcs) @pypi(python="3.11.9", packages={"mozmlops": "0.1.4"}) - @kubernetes + #@kubernetes + @nvct @step def upload_model_to_gcs(self): from mozmlops.cloud_storage_api_client import CloudStorageAPIClient @@ -211,7 +214,8 @@ def upload_model_to_gcs(self): ) self.next(self.end) - @kubernetes + #@kubernetes + @nvct @step def end(self): print( From 7807e306523c10d154d3625b867a5e8cdaa2a332 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 23 Oct 2025 13:01:58 +0200 Subject: [PATCH 2/6] Stop using mozmlops dependency in flow and use latest outerbounds package - use wandb package explicitly in flow where needed - disabled uploading model to gcs - install outerbounds package instead of mozmlops via requirements file --- .../image_classifier/image_classifier_flow.py | 20 +++++++++---------- .../requirements.example_image_classifier.txt | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/image_classifier/image_classifier_flow.py b/examples/image_classifier/image_classifier_flow.py index 56d6797..bbfa9f9 100644 --- a/examples/image_classifier/image_classifier_flow.py +++ b/examples/image_classifier/image_classifier_flow.py @@ -63,7 +63,7 @@ def start(self): # Keep @nvct decorator before @step decorator else the flow fails @pypi( python="3.11.9", - packages={"torch": "2.4.1", "torchvision": "0.19.1", "mozmlops": "0.1.4"}, + packages={"torch": "2.4.1", "torchvision": "0.19.1", "wandb": "0.22.2"}, ) @nvct # @kubernetes @@ -197,21 +197,21 @@ def evaluate(self): ) self.next(self.upload_model_to_gcs) - @pypi(python="3.11.9", packages={"mozmlops": "0.1.4"}) + @pypi(python="3.11.9") #@kubernetes @nvct @step def upload_model_to_gcs(self): - from mozmlops.cloud_storage_api_client import CloudStorageAPIClient + #from mozmlops.cloud_storage_api_client import CloudStorageAPIClient - print("Uploading model to gcs") + print("Skipping Uploading model to gcs") # init client - storage_client = CloudStorageAPIClient( - project_name=GCS_PROJECT_NAME, bucket_name=GCS_BUCKET_NAME - ) - storage_client.store( - data=self.model_state_dict_bytes, storage_path=MODEL_STORAGE_PATH - ) + #storage_client = CloudStorageAPIClient( + # project_name=GCS_PROJECT_NAME, bucket_name=GCS_BUCKET_NAME + #) + #storage_client.store( + # data=self.model_state_dict_bytes, storage_path=MODEL_STORAGE_PATH + #) self.next(self.end) #@kubernetes diff --git a/examples/image_classifier/requirements.example_image_classifier.txt b/examples/image_classifier/requirements.example_image_classifier.txt index 3a0da7e..49fdd63 100644 --- a/examples/image_classifier/requirements.example_image_classifier.txt +++ b/examples/image_classifier/requirements.example_image_classifier.txt @@ -1 +1 @@ -mozmlops==0.1.4 +outerbounds[gcp]==0.10.19 From 6e79000ab83581f5ec564661426549ee925537f4 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Thu, 23 Oct 2025 13:30:20 +0200 Subject: [PATCH 3/6] Update torch and torchvision dependency versions in flow to latest - torchvision 0.24.0 - torch 2.9.0 --- examples/image_classifier/image_classifier_flow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/image_classifier/image_classifier_flow.py b/examples/image_classifier/image_classifier_flow.py index bbfa9f9..223373a 100644 --- a/examples/image_classifier/image_classifier_flow.py +++ b/examples/image_classifier/image_classifier_flow.py @@ -33,7 +33,7 @@ class ImageClassifierFlow(FlowSpec): default=True, ) - @pypi(python="3.11.9", packages={"torchvision": "0.19.1"}) + @pypi(python="3.11.9", packages={"torchvision": "0.24.0"}) @card(type="default") #@kubernetes @nvct @@ -63,7 +63,7 @@ def start(self): # Keep @nvct decorator before @step decorator else the flow fails @pypi( python="3.11.9", - packages={"torch": "2.4.1", "torchvision": "0.19.1", "wandb": "0.22.2"}, + packages={"torch": "2.9.0", "torchvision": "0.24.0", "wandb": "0.22.2"}, ) @nvct # @kubernetes @@ -151,8 +151,8 @@ def train(self): @pypi( python="3.11.9", packages={ - "torch": "2.4.1", - "torchvision": "0.19.1", + "torch": "2.9.0", + "torchvision": "0.24.0", }, ) # Check https://docs.metaflow.org/api/step-decorators/kubernetes for details on @kubernetes decorator From 00342152685d7e38cce0096fd1a7266050006575 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Fri, 24 Oct 2025 15:02:04 +0200 Subject: [PATCH 4/6] Switch from @pypi to @conda --- examples/image_classifier/README.md | 2 +- .../image_classifier/image_classifier_flow.py | 29 +++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/examples/image_classifier/README.md b/examples/image_classifier/README.md index 5ee2f85..dd1b9a8 100644 --- a/examples/image_classifier/README.md +++ b/examples/image_classifier/README.md @@ -27,7 +27,7 @@ 2. Set the right GCP project and bucket in the [flow](./image_classifier_flow.py) where you want to upload the trained model to. 3. Run this command on the terminal (replace `your_wandb_api_key` and `your_wandb_project` with your details in this command): ```sh - WANDB_API_KEY=your_wandb_api_key WANDB_PROJECT=your_wandb_project python image_classifier_flow.py --environment=pypi run --offline False + WANDB_API_KEY=your_wandb_api_key WANDB_PROJECT=your_wandb_project python image_classifier_flow.py --environment=conda run --offline False ``` You can track the progress of the flow run on Outerbounds UI (see the url in your terminal logs). diff --git a/examples/image_classifier/image_classifier_flow.py b/examples/image_classifier/image_classifier_flow.py index 223373a..0823ec8 100644 --- a/examples/image_classifier/image_classifier_flow.py +++ b/examples/image_classifier/image_classifier_flow.py @@ -12,6 +12,7 @@ environment, kubernetes, pypi, + conda, nvct, ) from metaflow.cards import Markdown @@ -33,7 +34,8 @@ class ImageClassifierFlow(FlowSpec): default=True, ) - @pypi(python="3.11.9", packages={"torchvision": "0.24.0"}) + #@pypi(python="3.11.9", packages={"torchvision": "0.24.0"}) + @conda(python="3.11.9", packages={"torchvision": "0.20.1"}) @card(type="default") #@kubernetes @nvct @@ -61,9 +63,13 @@ def start(self): # Train the network # Keep @nvct decorator before @step decorator else the flow fails - @pypi( + #@pypi( + # python="3.11.9", + # packages={"torch": "2.9.0", "torchvision": "0.24.0", "wandb": "0.22.2"}, + #) + @conda( python="3.11.9", - packages={"torch": "2.9.0", "torchvision": "0.24.0", "wandb": "0.22.2"}, + packages={"pytorch": "2.8.0", "torchvision": "0.20.1", "wandb": "0.22.2"}, ) @nvct # @kubernetes @@ -148,12 +154,16 @@ def train(self): self.next(self.evaluate) # Test the model on the test data - @pypi( + #@pypi( + # python="3.11.9", + # packages={ + # "torch": "2.9.0", + # "torchvision": "0.24.0", + # }, + #) + @conda( python="3.11.9", - packages={ - "torch": "2.9.0", - "torchvision": "0.24.0", - }, + packages={"pytorch": "2.8.0", "torchvision": "0.20.1"}, ) # Check https://docs.metaflow.org/api/step-decorators/kubernetes for details on @kubernetes decorator #@kubernetes(cpu=1, memory=4096) @@ -197,7 +207,8 @@ def evaluate(self): ) self.next(self.upload_model_to_gcs) - @pypi(python="3.11.9") + #@pypi(python="3.11.9") + @conda(python="3.11.9") #@kubernetes @nvct @step From 08b3b98fd9edca77433160ab2cf79b5952422716 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Fri, 24 Oct 2025 15:32:09 +0200 Subject: [PATCH 5/6] Switch to python version "3.10" in the flow --- examples/image_classifier/image_classifier_flow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/image_classifier/image_classifier_flow.py b/examples/image_classifier/image_classifier_flow.py index 0823ec8..f1efeef 100644 --- a/examples/image_classifier/image_classifier_flow.py +++ b/examples/image_classifier/image_classifier_flow.py @@ -35,7 +35,7 @@ class ImageClassifierFlow(FlowSpec): ) #@pypi(python="3.11.9", packages={"torchvision": "0.24.0"}) - @conda(python="3.11.9", packages={"torchvision": "0.20.1"}) + @conda(python="3.10", packages={"torchvision": "0.20.1"}) @card(type="default") #@kubernetes @nvct @@ -68,7 +68,7 @@ def start(self): # packages={"torch": "2.9.0", "torchvision": "0.24.0", "wandb": "0.22.2"}, #) @conda( - python="3.11.9", + python="3.10", packages={"pytorch": "2.8.0", "torchvision": "0.20.1", "wandb": "0.22.2"}, ) @nvct @@ -162,7 +162,7 @@ def train(self): # }, #) @conda( - python="3.11.9", + python="3.10", packages={"pytorch": "2.8.0", "torchvision": "0.20.1"}, ) # Check https://docs.metaflow.org/api/step-decorators/kubernetes for details on @kubernetes decorator @@ -208,7 +208,7 @@ def evaluate(self): self.next(self.upload_model_to_gcs) #@pypi(python="3.11.9") - @conda(python="3.11.9") + @conda(python="3.10") #@kubernetes @nvct @step From 94d4a3812bb1146db0b4be2e1be9792fdb9c2563 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Fri, 24 Oct 2025 15:45:39 +0200 Subject: [PATCH 6/6] Don't specify any step's package dependency versions --- examples/image_classifier/image_classifier_flow.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/image_classifier/image_classifier_flow.py b/examples/image_classifier/image_classifier_flow.py index f1efeef..fab4153 100644 --- a/examples/image_classifier/image_classifier_flow.py +++ b/examples/image_classifier/image_classifier_flow.py @@ -35,7 +35,8 @@ class ImageClassifierFlow(FlowSpec): ) #@pypi(python="3.11.9", packages={"torchvision": "0.24.0"}) - @conda(python="3.10", packages={"torchvision": "0.20.1"}) + #@conda(python="3.10", packages={"torchvision": "0.20.1"}) + @conda(python="3.10", packages={"torchvision": ""}) @card(type="default") #@kubernetes @nvct @@ -69,7 +70,8 @@ def start(self): #) @conda( python="3.10", - packages={"pytorch": "2.8.0", "torchvision": "0.20.1", "wandb": "0.22.2"}, + #packages={"pytorch": "2.8.0", "torchvision": "0.20.1", "wandb": "0.22.2"}, + packages={"pytorch": "", "torchvision": "", "wandb": ""}, ) @nvct # @kubernetes @@ -163,7 +165,8 @@ def train(self): #) @conda( python="3.10", - packages={"pytorch": "2.8.0", "torchvision": "0.20.1"}, + #packages={"pytorch": "2.8.0", "torchvision": "0.20.1"}, + packages={"pytorch": "", "torchvision": ""}, ) # Check https://docs.metaflow.org/api/step-decorators/kubernetes for details on @kubernetes decorator #@kubernetes(cpu=1, memory=4096)