Deprecating the requirements.txt for dependencies management

.github/workflows/build-and-publish.yaml

@@ -42,7 +42,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r setup_requirements.txt
+          python -m pip install -r tox
       - name: Build and test with tox
         run: tox -e ${{ matrix.python-version.tox }}
       - name: Build and check wheel package

.github/workflows/format.yml

@@ -21,20 +21,19 @@ on:
     branches: [ "main" ]
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r setup_requirements.txt
-      - name: Check Formatting
+          python -m pip install tox
+      - name: Check formatting
         run: tox -e fmt
       - name: Run pylint
         run: tox -e lint
-

.github/workflows/test.yaml

@@ -1,5 +1,4 @@
 name: Test
-
 on:
   push:
     branches: [ "main" ]
@@ -9,14 +8,20 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - setup: "3.9"
+            tox: "py39"
+          - setup: "3.10"
+            tox: "py310"
+          - setup: "3.11"
+            tox: "py311"
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          python -m pip install -r setup_requirements.txt
+          python -m pip install --upgrade pip
+          python -m pip install tox
       - name: Run unit tests
         run: tox -e py

CONTRIBUTING.md

@@ -38,9 +38,9 @@ Before sending pull requests, make sure your changes pass formatting, linting an
 #### Dependencies
 If additional new Python module dependencies are required, think about where to put them:
 
-- If they're required for fms-hf-tuning, then append them to the end of the requirements.txt file.
-- If they're optional dependencies for additional functionality, then put them in the pyproject.toml file like were done for [flash-attn](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L26) or [aim](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L27).
-- If it's an additional dependency for development, then add it to the [dev](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L25) dependencies.
+- If they're required for fms-hf-tuning, then append them to the [dependencies](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L28) in the pyproject.toml.
+- If they're optional dependencies for additional functionality, then put them in the pyproject.toml file like were done for [flash-attn](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L44) or [aim](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L45).
+- If it's an additional dependency for development, then add it to the [dev](https://github.com/foundation-model-stack/fms-hf-tuning/blob/main/pyproject.toml#L43) dependencies.
 
 #### Code Review
 
@@ -141,6 +141,17 @@ We use [pytest](https://docs.pytest.org/) framework to run unit tests. The frame
 > Optionally, run `make test` command to do formatting, linting, and testing at once.
 </details>
 
+<details>
+<summary>Build wheel</summary>
+
+To build a wheel file:
+```shell
+tox -e build
+```
+Running the command will create a single ZIP-format archive containing the library source code with the .whl extension in the `dist/` directory.
+
+</details>
+
 ## Your First Code Contribution
 
 Unsure where to begin contributing? You can start by looking through these issues:

pyproject.toml

@@ -2,11 +2,12 @@
 build-backend = "setuptools.build_meta"
 requires = [
     "setuptools>=60",
-    "setuptools-scm>=8.0"]
+    "setuptools-scm>=8.0"
+    ]
 
 [project]
 name = "fms-hf-tuning"
-dynamic = ["version", "dependencies"]
+dynamic = ["version"]
 description = "FMS HF Tuning"
 authors = [
   {name = "Sukriti Sharma", email = "sukriti.sharma4@ibm.com"},
@@ -24,15 +25,25 @@ classifiers=[
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
 ]
+dependencies = [
+"numpy",
+"accelerate>=0.20.3",
+"transformers",
+"torch",
+"sentencepiece",
+"tokenizers>=0.13.3",
+"tqdm",
+"trl",
+"peft>=0.8.0",
+"datasets>=2.15.0",
+"fire"
+]
 
 [project.optional-dependencies]
 dev = ["wheel", "packaging", "ninja", "scikit-learn>=1.0, <2.0"]
 flash-attn = ["flash-attn"]
 aim = ["aim==3.18.1"]
 
-[tool.setuptools.dynamic]
-dependencies = {file = ["requirements.txt"]}
-
 [tool.setuptools.packages.find]
 exclude = ["tests", "tests.*"]
 namespaces = false

tox.ini

@@ -5,22 +5,22 @@ envlist = py, lint, fmt
 description = run unit tests
 deps =
     pytest>=7
-    -r requirements.txt
-    wheel>=0.38.4
 commands =
     pytest {posargs:tests}
 
 [testenv:fmt]
 description = format with pre-commit
+deps =
+    pre-commit
 commands = ./scripts/fmt.sh
 allowlist_externals = ./scripts/fmt.sh
 
 [testenv:lint]
 description = lint with pylint
-deps = pylint>=2.16.2,<=3.1.0
-        pytest
-        .[dev]
-        -r requirements.txt
+deps =
+    pylint>=2.16.2,<=3.1.0
+    pytest
+    .[dev]
 commands = pylint tuning scripts/*.py build/*.py tests
 allowlist_externals = pylint