Feature/upgrade lib multi channels

.github/workflows/python-test.yml

@@ -0,0 +1,79 @@
+name: Python package
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.10', '3.11']
+
+    steps:
+      # 1. Checkout the repository
+      - uses: actions/checkout@v4
+
+      # 2. Cache Conda packages (optional but recommended)
+      - name: Cache Conda packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.conda/pkgs
+          key: ${{ runner.os }}-conda-${{ hashFiles('**/environment.yml') }}
+          restore-keys: |
+            ${{ runner.os }}-conda-
+
+      # 3. Set up Miniconda
+      - name: Set up Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: true
+          activate-environment: ''
+
+      # 4. Create and activate Conda environment
+      - name: Create and activate Conda environment
+        shell: bash -l {0}
+        run: |
+          if [[ "${{ matrix.python-version }}" == pypy* ]]; then
+            conda create -n test-env pypy=3.10 -y
+          else
+            conda create -n test-env python=${{ matrix.python-version }} -y
+          fi
+          source activate test-env
+          pip install --upgrade pip
+
+      # 5. Display Python version
+      - name: Display Python version
+        shell: bash -l {0}
+        run: |
+          source activate test-env
+          python -c "import sys; print(sys.version)"
+
+      # 6. Install the package
+      - name: Install pyphoon2 library
+        shell: bash -l {0}
+        run: |
+          source activate test-env
+          pip install .
+
+      # 7. Verify installation
+      - name: Check if pyphoon2 was installed
+        shell: bash -l {0}
+        run: |
+          source activate test-env
+          pip list | grep pyphoon2
+
+      # 8. Verify PyPy installation (optional)
+      - name: Verify PyPy installation
+        if: startsWith(matrix.python-version, 'pypy')
+        shell: bash -l {0}
+        run: |
+          source activate test-env
+          pypy --version
+
+      # 9. Run tests
+      - name: Run tests
+        shell: bash -l {0}
+        run: |
+          source activate test-env
+          cd tests
+          # sh run_all_tests.sh

.gitignore

@@ -1,8 +1,9 @@
 # Dataset files
 *.tsv
 *.h5
-
+*.zip
 # Byte-compiled / optimized / DLL files
+tests/test_data_files
 __pycache__/
 *.py[cod]
 *$py.class

docs/conf.py

@@ -6,17 +6,18 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
+import os
+import sys
 project = 'pyphoon2'
-copyright = '2023, Jared Hwang'
-author = 'Jared Hwang'
-release = '1.0.0'
+copyright = '2023, Jared Hwang. 2024, Ngoc Anh Tong'
+author = 'Jared Hwang, Ngoc Anh Tong'
+release = '2.0.0'
 
 
 templates_path = ['_templates']
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 
 
-
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
@@ -25,8 +26,6 @@
 
 
 # Path setup
-import os 
-import sys
 sys.path.insert(0, os.path.abspath('../'))
 sys.path.insert(0, os.path.abspath('../pyphoon2'))
 
@@ -38,7 +37,7 @@
     'sphinx.ext.autodoc',  # Core library for html generation from docstrings
     'sphinx.ext.autosummary',  # Create neat summary tables
 ]
-autosummary_generate = True 
+autosummary_generate = True
 autodoc_member_order = 'bysource'
 
 autodoc_default_options = {

docs/frame_model.py

@@ -1,3 +1,9 @@
+# autopep8: off
+import os
+import sys
+import os.path
+sys.path.insert(0, os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..')))
 import torch
 from torch import nn
 from torch import optim
@@ -10,7 +16,7 @@
 from pathlib import Path
 from torch.utils.data import DataLoader
 
-from DigitalTyphoonDataloader.DigitalTyphoonDataset import DigitalTyphoonDataset
+from pyphoon2.DigitalTyphoonDataset import DigitalTyphoonDataset
 
 def main(args):
 
@@ -112,7 +118,7 @@ def transform_func(image_ray):
 
             # Calculate the loss
             loss = criterion(predictions, labels)
-        
+
             # backward pass
             loss.backward()
             # update weights

docs/model_example_1.rst

@@ -24,7 +24,7 @@ The Code
 	from pathlib import Path
 	from torch.utils.data import DataLoader
 
-	from DigitalTyphoonDataloader.DigitalTyphoonDataset import DigitalTyphoonDataset
+	from pyphoon2.DigitalTyphoonDataset import DigitalTyphoonDataset import DigitalTyphoonDataset
 
 	def main(args):
 

docs/model_example_2.rst

@@ -19,7 +19,7 @@ The Code
 	from pathlib import Path
 	from torch.utils.data import DataLoader
 
-	from DigitalTyphoonDataloader.DigitalTyphoonDataset import DigitalTyphoonDataset
+	from pyphoon2.DigitalTyphoonDataset import DigitalTyphoonDataset import DigitalTyphoonDataset
 
 	def main(args):
 

docs/padlabels.py

@@ -11,19 +11,20 @@
 from pathlib import Path
 from torch.utils.data import DataLoader
 
-from DigitalTyphoonDataloader.DigitalTyphoonDataset import DigitalTyphoonDataset
+from pyphoon2.DigitalTyphoonDataset import DigitalTyphoonDataset import DigitalTyphoonDataset
+
 
 def main(args):
 
-    ## Prepare the data
+    # Prepare the data
 
     # Specify the paths to the data
     data_path = args.dataroot
-    images_path = data_path + '/image/' # to the image folder
-    metadata_path = data_path + '/metadata/' # to the metadata folder
+    images_path = data_path + '/image/'  # to the image folder
+    metadata_path = data_path + '/metadata/'  # to the metadata folder
     json_path = data_path + '/metadata.json'  # to the metadata json
 
-    # Define a filter to pass to the loader. 
+    # Define a filter to pass to the loader.
     #     Any image that the function returns true will be included
     def image_filter(image):
         return image.grade() < 7
@@ -33,17 +34,22 @@ def image_filter(image):
     # So, image-by-image transforms (i.e. clipping, downsampling, etc. can/should be done here)
     def transform_func(image_ray):
         # Clip the pixel values between 150 and 350
-        image_ray = np.clip(image_ray, standardize_range[0], standardize_range[1])
+        image_ray = np.clip(
+            image_ray, standardize_range[0], standardize_range[1])
 
         # Standardize the pixel values between 0 and 1
-        image_ray = (image_ray - standardize_range[0]) / (standardize_range[1] - standardize_range[0])
+        image_ray = (
+            image_ray - standardize_range[0]) / (standardize_range[1] - standardize_range[0])
 
         # Downsample the images to 224, 224
         if downsample_size != (512, 512):
             image_ray = torch.Tensor(image_ray)
-            image_ray = torch.reshape(image_ray, [1, 1, image_ray.size()[0], image_ray.size()[1]])
-            image_ray = nn.functional.interpolate(image_ray, size=downsample_size, mode='bilinear', align_corners=False)
-            image_ray = torch.reshape(image_ray, [image_ray.size()[2], image_ray.size()[3]])
+            image_ray = torch.reshape(
+                image_ray, [1, 1, image_ray.size()[0], image_ray.size()[1]])
+            image_ray = nn.functional.interpolate(
+                image_ray, size=downsample_size, mode='bilinear', align_corners=False)
+            image_ray = torch.reshape(
+                image_ray, [image_ray.size()[2], image_ray.size()[3]])
             image_ray = image_ray.numpy()
         return image_ray
 
@@ -52,42 +58,42 @@ def transform_func(image_ray):
                                     str(metadata_path),
                                     str(json_path),
                                     'grade',  # the labels we'd like to retrieve from the dataset
-                                    get_images_by_sequence=True, # indicate we want typhoon sequences returned
-                                    filter_func=image_filter, # the filter function defined above
-                                    transform_func=transform_func, # the transform function defined above
-                                    transform=transforms.Compose([ # pytorch transform to apply to data before returning data
+                                    get_images_by_sequence=True,  # indicate we want typhoon sequences returned
+                                    filter_func=image_filter,  # the filter function defined above
+                                    transform_func=transform_func,  # the transform function defined above
+                                    transform=transforms.Compose([  # pytorch transform to apply to data before returning data
                                         PadSequence(505),
                                     ]),
                                     verbose=False)
 
-
     # Split the dataset into a training and test split (80% and 20% respectively)
-    #   split by sequence so all images in one sequence will belong to the same bucket 
+    #   split by sequence so all images in one sequence will belong to the same bucket
     train_set, test_set = dataset.random_split([0.8, 0.2], split_by='sequence')
 
     # Make Pytorch DataLoaders out of the returned sets. From here, it retains all Pytorch functionality.
-    trainloader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers)
-    testloader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers)
-
+    trainloader = DataLoader(
+        train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers)
+    testloader = DataLoader(test_set, batch_size=batch_size,
+                            shuffle=False, num_workers=num_workers)
 
-    ## Prepare the model
+    # Prepare the model
     # Hyperparameters
     num_epochs = args.max_epochs
     batch_size = 16
     learning_rate = 0.001
     standardize_range = (150, 350)
     downsample_size = (224, 224)
 
-    # Create a dummy model that will take input of size (505, 224, 224) (seq length, height, width) and output 
+    # Create a dummy model that will take input of size (505, 224, 224) (seq length, height, width) and output
     # a value for each image in the sequence (shape (505))
     linear1 = nn.Linear(224, 1)
 
     # Loss and optimizer
     criterion = nn.CrossEntropyLoss()
-    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)
+    optimizer = torch.optim.SGD(
+        model.parameters(), lr=learning_rate, momentum=0.9)
 
-
-    ## Train the model
+    # Train the model
     for epoch in np.arange(max_epochs):
 
         batches_per_epoch = len(trainloader)
@@ -99,11 +105,12 @@ def transform_func(image_ray):
 
             # Data is a tuple, with sequence in data[0] and labels in data[1]
             # sequence is shape (16, 505, 224, 224) (batch size, seq len, height, width)
-            # labels size is (16, 505) as we padded them as well. 
+            # labels size is (16, 505) as we padded them as well.
             sequence, labels = data
 
             # cast pixels to float and grade (label) to long
-            sequence, labels = torch.Tensor(sequence).float(), torch.Tensor(labels).long()
+            sequence, labels = torch.Tensor(
+                sequence).float(), torch.Tensor(labels).long()
 
             optimizer.zero_grad()
 
@@ -113,7 +120,7 @@ def transform_func(image_ray):
 
             # Calculate the loss
             loss = criterion(predictions, labels)
-        
+
             # backward pass
             loss.backward()
             # update weights
@@ -141,8 +148,8 @@ def __call__(self, received_sample):
         sample = torch.cat((pad, sample), dim=0)
 
         # Resize to (length, height, width)
-        sample = torch.reshape(sample, [sample.size()[0], sample.size()[1], sample.size()[2]])
-
+        sample = torch.reshape(
+            sample, [sample.size()[0], sample.size()[1], sample.size()[2]])
 
         # Do the same for the labels
         labels = torch.Tensor(labels)
@@ -154,8 +161,9 @@ def __call__(self, received_sample):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Train a resnet model')
-    parser.add_argument('--dataroot', required=True, type=str, help='path to the root data directory')
+    parser.add_argument('--dataroot', required=True, type=str,
+                        help='path to the root data directory')
     parser.add_argument('--maxepochs', default=100, type=int)
     args = parser.parse_args()
 
-    main(args)
+    main(args)