From 3410b779a60b80acf1809d8e4a0c8360cf2e05e7 Mon Sep 17 00:00:00 2001 From: menglutao Date: Thu, 9 Nov 2023 01:11:44 +0200 Subject: [PATCH 1/7] Add style check workflow file --- .github/workflows/style.yaml | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/style.yaml diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml new file mode 100644 index 00000000..ecc5939c --- /dev/null +++ b/.github/workflows/style.yaml @@ -0,0 +1,45 @@ +name: Style Check + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + style: + name: Lint with black, flake8, and isort + runs-on: ubuntu-latest + + steps: + - name: Check out source code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.6.9' + + - name: Cache pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install black==22.3.0 flake8==4.0.1 isort==5.10.1 + + - name: Run isort + run: isort . --check-only + + - name: Run black + run: black . --check + + - name: Run flake8 + run: flake8 . From 2ca49c843a1893448c159015b42798fd17e85637 Mon Sep 17 00:00:00 2001 From: menglutao Date: Thu, 9 Nov 2023 01:29:54 +0200 Subject: [PATCH 2/7] Add tests workflow file --- .github/workflows/style.yaml | 2 +- .github/workflows/tests.yaml | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml index ecc5939c..6dc21719 100644 --- a/.github/workflows/style.yaml +++ b/.github/workflows/style.yaml @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.6.9' + python-version: '3.8.10' - name: Cache pip dependencies uses: actions/cache@v3 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 00000000..75f3f5e5 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,39 @@ +name: Tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + name: Run pytest + runs-on: ubuntu-latest + + steps: + - name: Check out source code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8.10' + + - name: Cache Python dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run pytest with coverage + run: pytest -ra --cov=. --cov-report term-missing From 0ebe4d9205090398ea2f28a29901c304efd1ec45 Mon Sep 17 00:00:00 2001 From: menglutao Date: Fri, 10 Nov 2023 00:33:44 +0200 Subject: [PATCH 3/7] Add type conversion for compatibility --- flsim/utils/example_utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/flsim/utils/example_utils.py b/flsim/utils/example_utils.py index 9f12d8f4..b8faf73f 100644 --- a/flsim/utils/example_utils.py +++ b/flsim/utils/example_utils.py @@ -28,8 +28,14 @@ from tqdm import tqdm -def collate_fn(batch: Tuple) -> Dict[str, Any]: - feature, label = batch +def collate_fn(batch: Any) -> Dict[str, Any]: + if isinstance(batch, tuple): + feature,label = batch + elif isinstance(batch, dict): + feature = batch['image'] + label = batch['label'] + else: + raise TypeError("The batch must be a tuple or dict") return {"features": feature, "labels": label} @@ -163,7 +169,13 @@ def get_num_examples(batch: List) -> int: def fl_training_batch( features: List[torch.Tensor], labels: List[float] ) -> Dict[str, torch.Tensor]: - return {"features": torch.stack(features), "labels": torch.Tensor(labels)} + # Check the type of the first element in labels list to determine if conversion is needed + if not isinstance(labels[0],torch.Tensor): + labels = torch.tensor(labels, dtype=torch.float32) + else: + labels = torch.stack(labels) + + return {"features": torch.stack(features), "labels": labels} class LEAFDataLoader(IFLDataLoader): From edee290666886ab92c4e6bbee81fc292e26b1544 Mon Sep 17 00:00:00 2001 From: menglutao Date: Fri, 10 Nov 2023 00:41:51 +0200 Subject: [PATCH 4/7] Update python version --- .github/workflows/style.yaml | 2 +- .github/workflows/tests.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml index 6dc21719..2459cd4c 100644 --- a/.github/workflows/style.yaml +++ b/.github/workflows/style.yaml @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8.10' + python-version: '3.8.12' - name: Cache pip dependencies uses: actions/cache@v3 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 75f3f5e5..7bcf4b80 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8.10' + python-version: '3.8.12' - name: Cache Python dependencies uses: actions/cache@v3 @@ -34,6 +34,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - + - name: Run pytest with coverage run: pytest -ra --cov=. --cov-report term-missing From bb4a77979e3efbc054c71765d5eace6a9ebe17ad Mon Sep 17 00:00:00 2001 From: menglutao Date: Fri, 10 Nov 2023 00:57:59 +0200 Subject: [PATCH 5/7] Remove workflow files --- .github/workflows/style.yaml | 45 ------------------------------------ .github/workflows/tests.yaml | 39 ------------------------------- 2 files changed, 84 deletions(-) delete mode 100644 .github/workflows/style.yaml delete mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml deleted file mode 100644 index 2459cd4c..00000000 --- a/.github/workflows/style.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Style Check - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - style: - name: Lint with black, flake8, and isort - runs-on: ubuntu-latest - - steps: - - name: Check out source code - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8.12' - - - name: Cache pip dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install black==22.3.0 flake8==4.0.1 isort==5.10.1 - - - name: Run isort - run: isort . --check-only - - - name: Run black - run: black . --check - - - name: Run flake8 - run: flake8 . diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml deleted file mode 100644 index 7bcf4b80..00000000 --- a/.github/workflows/tests.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: Tests - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build: - name: Run pytest - runs-on: ubuntu-latest - - steps: - - name: Check out source code - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8.12' - - - name: Cache Python dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Run pytest with coverage - run: pytest -ra --cov=. --cov-report term-missing From 241ce27c21b44d66b44cb72de798628bdc181be5 Mon Sep 17 00:00:00 2001 From: menglutao Date: Tue, 21 Nov 2023 13:45:39 +0200 Subject: [PATCH 6/7] apply lint --- flsim/utils/example_utils.py | 37 +++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/flsim/utils/example_utils.py b/flsim/utils/example_utils.py index b8faf73f..87afc59e 100644 --- a/flsim/utils/example_utils.py +++ b/flsim/utils/example_utils.py @@ -8,10 +8,16 @@ # utils for use in the examples and tutorials import random -from typing import Any, Dict, Generator, Iterable, Iterator, List, Optional, Tuple +from typing import Any, Dict, Generator, Iterable, Iterator, List, Optional import torch import torch.nn.functional as F +from tqdm import tqdm +from torchvision import transforms +from torch import nn +from torch.utils.data import Dataset +from torchvision.datasets.cifar import CIFAR10 +from torchvision.datasets.vision import VisionDataset from flsim.data.data_provider import IFLDataProvider, IFLUserData from flsim.data.data_sharder import FLDataSharder, SequentialSharder from flsim.interfaces.data_loader import IFLDataLoader @@ -20,20 +26,19 @@ from flsim.metrics_reporter.tensorboard_metrics_reporter import FLMetricsReporter from flsim.utils.data.data_utils import batchify from flsim.utils.simple_batch_metrics import FLBatchMetrics -from torch import nn -from torch.utils.data import Dataset -from torchvision import transforms -from torchvision.datasets.cifar import CIFAR10 -from torchvision.datasets.vision import VisionDataset -from tqdm import tqdm + + + + + def collate_fn(batch: Any) -> Dict[str, Any]: if isinstance(batch, tuple): - feature,label = batch + feature, label = batch elif isinstance(batch, dict): - feature = batch['image'] - label = batch['label'] + feature = batch["image"] + label = batch["label"] else: raise TypeError("The batch must be a tuple or dict") return {"features": feature, "labels": label} @@ -169,8 +174,8 @@ def get_num_examples(batch: List) -> int: def fl_training_batch( features: List[torch.Tensor], labels: List[float] ) -> Dict[str, torch.Tensor]: - # Check the type of the first element in labels list to determine if conversion is needed - if not isinstance(labels[0],torch.Tensor): + # Check the type of the first element in labels list to determine if conversion is needed + if not isinstance(labels[0], torch.Tensor): labels = torch.tensor(labels, dtype=torch.float32) else: labels = torch.stack(labels) @@ -242,10 +247,9 @@ def num_train_users(self) -> int: def get_train_user(self, user_index: int) -> IFLUserData: if user_index in self._train_users: return self._train_users[user_index] - else: - raise IndexError( - f"Index {user_index} is out of bound for list with len {self.num_train_users()}" - ) + raise IndexError( + f"Index {user_index} is out of bound for list with len {self.num_train_users()}" + ) def train_users(self) -> Iterable[IFLUserData]: for user_data in self._train_users.values(): @@ -273,7 +277,6 @@ def _create_fl_users( def build_data_provider( local_batch_size, examples_per_user, image_size ) -> DataProvider: - # 1. Create training, eval, and test datasets like in non-federated learning. transform = transforms.Compose( [ From d7d33e44cdac18a075e00bf401c108fe82426281 Mon Sep 17 00:00:00 2001 From: menglutao Date: Tue, 21 Nov 2023 13:55:22 +0200 Subject: [PATCH 7/7] lint --- flsim/utils/example_utils.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/flsim/utils/example_utils.py b/flsim/utils/example_utils.py index 87afc59e..065d49dd 100644 --- a/flsim/utils/example_utils.py +++ b/flsim/utils/example_utils.py @@ -27,13 +27,16 @@ from flsim.utils.data.data_utils import batchify from flsim.utils.simple_batch_metrics import FLBatchMetrics +def collate_fn(batch: Any) -> Dict[str, Any]: + """ + Process the given batch and return a dictionary with features and labels. + Args: + batch (Any): The batch to process, can be a tuple or a dictionary. - - - - -def collate_fn(batch: Any) -> Dict[str, Any]: + Returns: + Dict[str, Any]: A dictionary containing the processed features and labels. + """ if isinstance(batch, tuple): feature, label = batch elif isinstance(batch, dict): @@ -45,6 +48,15 @@ def collate_fn(batch: Any) -> Dict[str, Any]: class DataLoader(IFLDataLoader): + + """Data loader class for federated learning. + + This data loader handles the batching and sharding of datasets for + federated learning scenarios. + + Attributes: + SEED (int): Seed for random number generation. + """ SEED = 2137 random.seed(SEED)