Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.ruff_cache
5 changes: 2 additions & 3 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ LINHUB_ORG=57512466-a2eb-414b-a332-ffd5486a6fb1
LINHUB_TASK=a5040509-de9c-4757-8cb3-9087b5191a2e
LINHUB_ENTITY=07a2fdef-d260-4f30-a350-8a7afc2af2a5
PKG_NAME=wsd
PJ_DIR=$PWD
COMPOSE_FILE=./docker/vanilla/compose.yml:./compose.yml
FORCE_PRE_COMMIT=true
FORCE_DOWNLOAD=true
FORCE_DOWNLOAD=false
PJ_DIR=$PWD
35 changes: 5 additions & 30 deletions .github/workflows/trigger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
branches: ["main"]

jobs:
build_only:
build_linux:
if: github.event_name == 'push'
runs-on: ubuntu-latest
outputs:
Expand All @@ -25,19 +25,9 @@ jobs:
# Install task
sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin

# Install uv
task install_uv
# Install the package
task install.package

# Set python version
uv venv --python=3.10
source .venv/bin/activate

# Install pre-commit
uv pip install pre-commit
pre-commit install

# Build
task build
shell: bash

build_mac:
Expand All @@ -56,22 +46,7 @@ jobs:
# Install task
sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin

# Install uv
task install_uv

# Set python version
uv venv --python=3.10 && source .venv/bin/activate

# Install pre-commit
uv pip install pre-commit
pre-commit install

# Build
task build.wheel

# Install package
sudo mkdir /tmp/wheels
sudo cp dist/*.whl /tmp/wheels/
sudo uv pip install $(ls /tmp/wheels | sed 's|^|/tmp/wheels/|') --system --break-system-packages
# Install the package
task install.package

shell: bash
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Fast, efficient, open source & open data Word Sense Disambiguation models for pr
The easiest way to install `wsd` is to use pip:

```
pip install wsd
pip install wsd[all]
```

You will also need the [The JMDict Project](https://www.edrdg.org/jmdict/j_jmdict.html) dictionary. You can use the following helper to download the file:
Expand Down
36 changes: 9 additions & 27 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,16 @@ env:
sh: grep 'version =' pyproject.toml | cut -d '"' -f 2
RSBUILD:
sh: if command -v rsbuild > /dev/null; then echo "true"; else echo "false"; fi
PJ_DIR:
sh: echo $PWD
JMDICT: "https://drive.usercontent.google.com/download?id=1dlvguHuMjDmtpA4beu1WaVbDte5j4SLk&confirm=yy"
XLWSD: "https://drive.usercontent.google.com/download?id=1d75OUrM3dyAvYUsnBXle-Z1W0NHyKIBD&confirm=yy"
VANILLA: docker run --rm
-v ${PWD}:/workspace
-v ${PJ_DIR}:/workspace
--workdir /workspace
--entrypoint bash
-i asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 -c

DEV: docker run --rm
-v ${PWD}:/workspace
-v ${PJ_DIR}:/workspace
--workdir /workspace
--entrypoint bash
-i asia.gcr.io/linalgo/${PKG_NAME}:dev
Expand Down Expand Up @@ -55,18 +53,6 @@ tasks:
cmds:
- 'echo "rsbuild is missing"'

install_uv:
desc: "Install the uv package if not already installed"
cmds:
- |
if ! which uv > /dev/null 2>&1; then
echo "Installing uv...";
curl -LsSf https://astral.sh/uv/install.sh | sh;
sudo mv $HOME/.local/bin/uv /usr/local/bin/
else
echo "uv is already installed.";
fi

add_build.mac:
cmds:
- sed -i '' 's/__version__ = ".*"/__version__ = "{{.VERSION}}"/' "${PKG_NAME}/__init__.py";
Expand Down Expand Up @@ -94,23 +80,13 @@ tasks:
# Build Tasks
build:
desc: "Tasks related to building the project"
deps: [install_uv, add_build]
deps: [add_build]
cmds:
- if [ "{{.FORCE_DOWNLOAD}}" = "true" ]; then task download; fi
- task build.docker

build.wheel:
desc: "Build the project wheel package"
deps: [install_uv, add_build]
cmds:
- if [ "{{.FORCE_PRE_COMMIT}}" = "true" ]; then pre-commit run -a; fi
- "mkdir -p dist/legacy && mv dist/*.whl dist/legacy/ || true"
- uv build --wheel
- task clean

build.docker:
desc: "Build the Docker image for the project"
deps: [build.wheel]
cmds:
- docker compose build vanilla
- docker compose build
Expand All @@ -137,3 +113,9 @@ tasks:
dataset:
cmds:
- gcloud auth application-default login

install.package:
cmds:
- pip install -U pre-commit --break-system-packages # Install pre-commit
- pre-commit install
- pip install -U wsd[all] --break-system-packages # Install the package
23 changes: 16 additions & 7 deletions compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,40 @@ x-default: &default
tty: true
restart: always

x-dev: &dev
x-mount_local: &mount_local
user: foo
env_file:
- .env
entrypoint: bash
volumes:
- ./:/workspace
- $PJ_DIR:/workspace

services:
sandbox:
image: asia.gcr.io/linalgo/${PKG_NAME}
<<: [ *default ]
build:
context: ${PJ_DIR}
context: $PJ_DIR
dockerfile: docker/sandbox/Dockerfile
args:
- PY_VER=310
- EXTRA=all

dev:
image: asia.gcr.io/linalgo/${PKG_NAME}:dev
container_name: wsd-dev
<<: [ *default, *dev ]
<<: [ *default ]
build:
context: ${PJ_DIR}
context: $PJ_DIR
dockerfile: docker/sandbox/Dockerfile
args:
- PY_VER=310
- EXTRA=dev
volumes:
- ${PJ_DIR}:/workspace
vanilla:
image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310
build:
context: $PJ_DIR
dockerfile: docker/vanilla/Dockerfile
args:
- PY_TAG=3.10-slim
target: dev
39 changes: 6 additions & 33 deletions docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,6 @@ Before you begin, `deactivate your virtual environment if any` and ensure you h
which task
```

- [uv](https://docs.astral.sh/uv/getting-started/installation/)

- Install the `uv` package if not already installed:

```bash
curl -LsSf https://astral.sh/uv/install.sh | sh
sudo mv $HOME/.local/bin/uv /usr/local/bin/
```

- Make sure the following command doesn't return an error

```bash
which uv
```

## Installation

To set up the project, follow these steps:
Expand All @@ -75,34 +60,22 @@ To set up the project, follow these steps:
cp .env.example .env
```

3. **Create the virtual environment with uv**:

```bash
uv venv --python=3.10 && source .venv/bin/activate
```

4. **Install [pre-commit](https://pre-commit.com/)**:
3. **Install [pre-commit](https://pre-commit.com/)**:

If `pre-commit` is not already installed, you can install it using the following command:

```bash
uv pip install pre-commit
pre-commit run -a
```

5. **Build the wheel**:

```bash
task build.wheel
pip install pre-commit
pre-commit install
```

6. **Build docker image**:
4. **Build docker image**:

```bash
task build.docker
task build
```

7. **Access the Docker Container**:
5. **Access the Docker Container**:

```bash
docker compose up dev -d
Expand Down
10 changes: 7 additions & 3 deletions docker/sandbox/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ FROM asia.gcr.io/linalgo/wsd:vanilla-py${PY_VER}

ARG EXTRA

# Copy the project files
COPY pyproject.toml ./pyproject.toml
COPY .env ./.env
COPY README.md ./README.md
COPY wsd ./wsd

# Install the library
COPY dist/*.whl /tmp/wheels/
RUN uv pip install $(ls /tmp/wheels | sed 's|^|/tmp/wheels/|')[$EXTRA] --system && \
sudo rm /tmp/wheels/*.whl
RUN pip install wsd[$EXTRA] --break-system-packages && rm -r pyproject.toml README.md .env wsd

ENTRYPOINT [ "bash" ]
9 changes: 0 additions & 9 deletions docker/vanilla/compose.yml

This file was deleted.

14 changes: 12 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@ version = "0.0.1rc0"
description = "Multi-lingual Word Sense Disambiguation."
readme = "README.md"
requires-python = ">=3.10"
license = { text = "MIT" }
license = "MIT"
classifiers = [
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
]
authors = [
Expand All @@ -31,6 +30,17 @@ dependencies = [

[project.optional-dependencies]
dev = ["pytest", "pre-commit"]
all = [
"fugashi[unidic]",
"google-genai",
"linalgo==0.1.4",
"mesop",
"numpy",
"pillow",
"rich",
"scikit-learn",
"typer",
]

[project.urls]
Homepage = "https://github.com/linalgo/wsd"
Expand Down
2 changes: 1 addition & 1 deletion wsd/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Init file containing version and build"""

__version__ = "0.0.1rc0"
__build__ = "Wed Mar 26 20:57:54 JST 2025"
__build__ = "Tue May 6 11:02:04 PM JST 2025"