From c59d136ba7e26523cc00247afbce5061401ae13c Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Mon, 28 Apr 2025 13:00:32 +0900 Subject: [PATCH 1/9] remove wheel file and uv --- .dockerignore | 1 + .env.example | 2 +- .github/workflows/trigger.yml | 28 +++++-------------------- README.md | 2 +- Taskfile.yml | 28 +++---------------------- compose.yml | 13 ++++++------ docker/README.md | 39 ++++++----------------------------- docker/sandbox/Dockerfile | 10 ++++++--- docker/vanilla/compose.yml | 2 +- pyproject.toml | 14 +++++++++++-- wsd/__init__.py | 2 +- 11 files changed, 45 insertions(+), 96 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b25701b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.ruff_cache diff --git a/.env.example b/.env.example index 6eb897f..e16cd64 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ LINHUB_ORG=57512466-a2eb-414b-a332-ffd5486a6fb1 LINHUB_TASK=a5040509-de9c-4757-8cb3-9087b5191a2e LINHUB_ENTITY=07a2fdef-d260-4f30-a350-8a7afc2af2a5 PKG_NAME=wsd -PJ_DIR=$PWD +# PJ_DIR=$PWD COMPOSE_FILE=./docker/vanilla/compose.yml:./compose.yml FORCE_PRE_COMMIT=true FORCE_DOWNLOAD=true diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml index a2f8c1c..a7210b2 100644 --- a/.github/workflows/trigger.yml +++ b/.github/workflows/trigger.yml @@ -7,7 +7,7 @@ on: branches: ["main"] jobs: - build_only: + build_linux: if: github.event_name == 'push' runs-on: ubuntu-latest outputs: @@ -25,18 +25,11 @@ jobs: # Install task sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin - # Install uv - task install_uv - - # Set python version - uv venv --python=3.10 - source .venv/bin/activate - # Install pre-commit - uv pip install pre-commit + pip install pre-commit pre-commit install - # Build + # Build (docker + wsd[all]) task build shell: bash @@ -56,22 +49,11 @@ jobs: # Install task sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin - # Install uv - task install_uv - - # Set python version - uv venv --python=3.10 && source .venv/bin/activate - # Install pre-commit - uv pip install pre-commit + pip install pre-commit pre-commit install # Build - task build.wheel - - # Install package - sudo mkdir /tmp/wheels - sudo cp dist/*.whl /tmp/wheels/ - sudo uv pip install $(ls /tmp/wheels | sed 's|^|/tmp/wheels/|') --system --break-system-packages + pip install wsd[all] --break-system-packages shell: bash diff --git a/README.md b/README.md index a0de3b4..9d58d80 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Fast, efficient, open source & open data Word Sense Disambiguation models for pr The easiest way to install `wsd` is to use pip: ``` -pip install wsd +pip install wsd[all] ``` You will also need the [The JMDict Project](https://www.edrdg.org/jmdict/j_jmdict.html) dictionary. You can use the following helper to download the file: diff --git a/Taskfile.yml b/Taskfile.yml index 7bf8b17..7e69651 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,8 +9,8 @@ env: sh: grep 'version =' pyproject.toml | cut -d '"' -f 2 RSBUILD: sh: if command -v rsbuild > /dev/null; then echo "true"; else echo "false"; fi - PJ_DIR: - sh: echo $PWD + # PJ_DIR: + # sh: echo $PWD JMDICT: "https://drive.usercontent.google.com/download?id=1dlvguHuMjDmtpA4beu1WaVbDte5j4SLk&confirm=yy" XLWSD: "https://drive.usercontent.google.com/download?id=1d75OUrM3dyAvYUsnBXle-Z1W0NHyKIBD&confirm=yy" VANILLA: docker run --rm @@ -55,18 +55,6 @@ tasks: cmds: - 'echo "rsbuild is missing"' - install_uv: - desc: "Install the uv package if not already installed" - cmds: - - | - if ! which uv > /dev/null 2>&1; then - echo "Installing uv..."; - curl -LsSf https://astral.sh/uv/install.sh | sh; - sudo mv $HOME/.local/bin/uv /usr/local/bin/ - else - echo "uv is already installed."; - fi - add_build.mac: cmds: - sed -i '' 's/__version__ = ".*"/__version__ = "{{.VERSION}}"/' "${PKG_NAME}/__init__.py"; @@ -94,23 +82,13 @@ tasks: # Build Tasks build: desc: "Tasks related to building the project" - deps: [install_uv, add_build] + deps: [add_build] cmds: - if [ "{{.FORCE_DOWNLOAD}}" = "true" ]; then task download; fi - task build.docker - build.wheel: - desc: "Build the project wheel package" - deps: [install_uv, add_build] - cmds: - - if [ "{{.FORCE_PRE_COMMIT}}" = "true" ]; then pre-commit run -a; fi - - "mkdir -p dist/legacy && mv dist/*.whl dist/legacy/ || true" - - uv build --wheel - - task clean - build.docker: desc: "Build the Docker image for the project" - deps: [build.wheel] cmds: - docker compose build vanilla - docker compose build diff --git a/compose.yml b/compose.yml index ac4429e..3084de4 100644 --- a/compose.yml +++ b/compose.yml @@ -3,31 +3,32 @@ x-default: &default tty: true restart: always -x-dev: &dev +x-mount_local: &mount_local user: foo entrypoint: bash volumes: - - ./:/workspace + - ${PJ_DIR}:/workspace services: sandbox: image: asia.gcr.io/linalgo/${PKG_NAME} - <<: [ *default ] + <<: [ *default, *mount_local ] build: context: ${PJ_DIR} dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 + - EXTRA=all + volumes: + - ${PJ_DIR}:/workspace dev: image: asia.gcr.io/linalgo/${PKG_NAME}:dev container_name: wsd-dev - <<: [ *default, *dev ] + <<: [ *default, *mount_local ] build: context: ${PJ_DIR} dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 - EXTRA=dev - volumes: - - ${PJ_DIR}:/workspace diff --git a/docker/README.md b/docker/README.md index 179063a..ed92758 100644 --- a/docker/README.md +++ b/docker/README.md @@ -43,21 +43,6 @@ Before you begin, `deactivate your virtual environment if any` and ensure you h which task ``` -- [uv](https://docs.astral.sh/uv/getting-started/installation/) - - - Install the `uv` package if not already installed: - - ```bash - curl -LsSf https://astral.sh/uv/install.sh | sh - sudo mv $HOME/.local/bin/uv /usr/local/bin/ - ``` - - - Make sure the following command doesn't return an error - - ```bash - which uv - ``` - ## Installation To set up the project, follow these steps: @@ -75,34 +60,22 @@ To set up the project, follow these steps: cp .env.example .env ``` -3. **Create the virtual environment with uv**: - - ```bash - uv venv --python=3.10 && source .venv/bin/activate - ``` - -4. **Install [pre-commit](https://pre-commit.com/)**: +3. **Install [pre-commit](https://pre-commit.com/)**: If `pre-commit` is not already installed, you can install it using the following command: ```bash - uv pip install pre-commit - pre-commit run -a - ``` - -5. **Build the wheel**: - - ```bash - task build.wheel + pip install pre-commit + pre-commit install ``` -6. **Build docker image**: +4. **Build docker image**: ```bash - task build.docker + task build ``` -7. **Access the Docker Container**: +5. **Access the Docker Container**: ```bash docker compose up dev -d diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile index 5f1aa81..8aa5870 100644 --- a/docker/sandbox/Dockerfile +++ b/docker/sandbox/Dockerfile @@ -3,9 +3,13 @@ FROM asia.gcr.io/linalgo/wsd:vanilla-py${PY_VER} ARG EXTRA +# Copy the project files +COPY pyproject.toml ./pyproject.toml +COPY .env ./.env +COPY README.md ./README.md +COPY wsd ./wsd + # Install the library -COPY dist/*.whl /tmp/wheels/ -RUN uv pip install $(ls /tmp/wheels | sed 's|^|/tmp/wheels/|')[$EXTRA] --system && \ - sudo rm /tmp/wheels/*.whl +RUN pip install .[$EXTRA] --break-system-packages && rm -r pyproject.toml README.md .env wsd ENTRYPOINT [ "bash" ] diff --git a/docker/vanilla/compose.yml b/docker/vanilla/compose.yml index 22bc19a..967a355 100644 --- a/docker/vanilla/compose.yml +++ b/docker/vanilla/compose.yml @@ -3,7 +3,7 @@ services: image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 build: context: ${PJ_DIR} - dockerfile: ./docker/vanilla/Dockerfile + dockerfile: docker/vanilla/Dockerfile args: - PY_TAG=3.10-slim target: dev diff --git a/pyproject.toml b/pyproject.toml index a7dbd9c..447bb7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,9 @@ version = "0.0.1rc0" description = "Multi-lingual Word Sense Disambiguation." readme = "README.md" requires-python = ">=3.10" -license = { text = "MIT" } +license = "MIT" classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] authors = [ @@ -31,6 +30,17 @@ dependencies = [ [project.optional-dependencies] dev = ["pytest", "pre-commit"] +all = [ + "fugashi[unidic]", + "google-genai", + "linalgo==0.1.4", + "mesop", + "numpy", + "pillow", + "rich", + "scikit-learn", + "typer", +] [project.urls] Homepage = "https://github.com/linalgo/wsd" diff --git a/wsd/__init__.py b/wsd/__init__.py index 073ea04..1f1436a 100644 --- a/wsd/__init__.py +++ b/wsd/__init__.py @@ -1,4 +1,4 @@ """Init file containing version and build""" __version__ = "0.0.1rc0" -__build__ = "Wed Mar 26 20:57:54 JST 2025" +__build__ = "Mon Apr 28 12:41:54 JST 2025" From d3ae9e2bb425f6d60ad361d08acb0859655bdba4 Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Mon, 28 Apr 2025 13:03:32 +0900 Subject: [PATCH 2/9] restore env --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index e16cd64..6eb897f 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ LINHUB_ORG=57512466-a2eb-414b-a332-ffd5486a6fb1 LINHUB_TASK=a5040509-de9c-4757-8cb3-9087b5191a2e LINHUB_ENTITY=07a2fdef-d260-4f30-a350-8a7afc2af2a5 PKG_NAME=wsd -# PJ_DIR=$PWD +PJ_DIR=$PWD COMPOSE_FILE=./docker/vanilla/compose.yml:./compose.yml FORCE_PRE_COMMIT=true FORCE_DOWNLOAD=true From 3d5112ef4711b71fa1853c16ef6368f3dc2e1b3f Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Mon, 28 Apr 2025 13:06:05 +0900 Subject: [PATCH 3/9] remove comment --- Taskfile.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 7e69651..00ef8e4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,8 +9,6 @@ env: sh: grep 'version =' pyproject.toml | cut -d '"' -f 2 RSBUILD: sh: if command -v rsbuild > /dev/null; then echo "true"; else echo "false"; fi - # PJ_DIR: - # sh: echo $PWD JMDICT: "https://drive.usercontent.google.com/download?id=1dlvguHuMjDmtpA4beu1WaVbDte5j4SLk&confirm=yy" XLWSD: "https://drive.usercontent.google.com/download?id=1d75OUrM3dyAvYUsnBXle-Z1W0NHyKIBD&confirm=yy" VANILLA: docker run --rm From 423da11f19a9f652dbad18c61a6587818a4527c9 Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Mon, 28 Apr 2025 13:07:50 +0900 Subject: [PATCH 4/9] remove comment --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 6eb897f..e4f926e 100644 --- a/.env.example +++ b/.env.example @@ -7,4 +7,4 @@ PKG_NAME=wsd PJ_DIR=$PWD COMPOSE_FILE=./docker/vanilla/compose.yml:./compose.yml FORCE_PRE_COMMIT=true -FORCE_DOWNLOAD=true +FORCE_DOWNLOAD=false From 199f8b1e0084b4fac976f6cd0a9eadff8a00a738 Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Tue, 6 May 2025 22:52:20 +0900 Subject: [PATCH 5/9] fix build --- compose.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/compose.yml b/compose.yml index 3084de4..285f7c8 100644 --- a/compose.yml +++ b/compose.yml @@ -7,28 +7,34 @@ x-mount_local: &mount_local user: foo entrypoint: bash volumes: - - ${PJ_DIR}:/workspace + - ./:/workspace services: sandbox: image: asia.gcr.io/linalgo/${PKG_NAME} <<: [ *default, *mount_local ] build: - context: ${PJ_DIR} + context: . dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 - EXTRA=all - volumes: - - ${PJ_DIR}:/workspace dev: image: asia.gcr.io/linalgo/${PKG_NAME}:dev container_name: wsd-dev <<: [ *default, *mount_local ] build: - context: ${PJ_DIR} + context: . dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 - EXTRA=dev + vanilla: + image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 + build: + context: . + dockerfile: docker/vanilla/Dockerfile + args: + - PY_TAG=3.10-slim + target: dev From 45c6e4f33bc819a7eef0132036ad69a5d7280876 Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Tue, 6 May 2025 22:55:55 +0900 Subject: [PATCH 6/9] fix ci --- .env.example | 3 +-- compose.yml | 10 ++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index e4f926e..029bb09 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,6 @@ LINHUB_ORG=57512466-a2eb-414b-a332-ffd5486a6fb1 LINHUB_TASK=a5040509-de9c-4757-8cb3-9087b5191a2e LINHUB_ENTITY=07a2fdef-d260-4f30-a350-8a7afc2af2a5 PKG_NAME=wsd -PJ_DIR=$PWD -COMPOSE_FILE=./docker/vanilla/compose.yml:./compose.yml FORCE_PRE_COMMIT=true FORCE_DOWNLOAD=false +PJ_DIR=$PWD diff --git a/compose.yml b/compose.yml index 285f7c8..8ca7e9d 100644 --- a/compose.yml +++ b/compose.yml @@ -5,16 +5,18 @@ x-default: &default x-mount_local: &mount_local user: foo + env_file: + - .env entrypoint: bash volumes: - - ./:/workspace + - $PJ_DIR:/workspace services: sandbox: image: asia.gcr.io/linalgo/${PKG_NAME} <<: [ *default, *mount_local ] build: - context: . + context: $PJ_DIR dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 @@ -25,7 +27,7 @@ services: container_name: wsd-dev <<: [ *default, *mount_local ] build: - context: . + context: $PJ_DIR dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 @@ -33,7 +35,7 @@ services: vanilla: image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 build: - context: . + context: $PJ_DIR dockerfile: docker/vanilla/Dockerfile args: - PY_TAG=3.10-slim From 89e51f240d8a8297c56c47b50b2298ac7737b298 Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Tue, 6 May 2025 22:57:30 +0900 Subject: [PATCH 7/9] fix ci: --- Taskfile.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Taskfile.yml b/Taskfile.yml index 00ef8e4..3f1d7a9 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,6 +3,8 @@ dotenv: [".env"] env: OS_TYPE: sh: uname + PJ_DIR: + sh: $PWD BUILD: sh: LC_TIME=en_US.UTF-8 date VERSION: From a58c718fa5157f491eab21cbfb624d9edbfebfda Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Tue, 6 May 2025 22:58:47 +0900 Subject: [PATCH 8/9] fix ci: --- Taskfile.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3f1d7a9..a75a311 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,8 +3,6 @@ dotenv: [".env"] env: OS_TYPE: sh: uname - PJ_DIR: - sh: $PWD BUILD: sh: LC_TIME=en_US.UTF-8 date VERSION: @@ -14,13 +12,13 @@ env: JMDICT: "https://drive.usercontent.google.com/download?id=1dlvguHuMjDmtpA4beu1WaVbDte5j4SLk&confirm=yy" XLWSD: "https://drive.usercontent.google.com/download?id=1d75OUrM3dyAvYUsnBXle-Z1W0NHyKIBD&confirm=yy" VANILLA: docker run --rm - -v ${PWD}:/workspace + -v ${PJ_DIR}:/workspace --workdir /workspace --entrypoint bash -i asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 -c DEV: docker run --rm - -v ${PWD}:/workspace + -v ${PJ_DIR}:/workspace --workdir /workspace --entrypoint bash -i asia.gcr.io/linalgo/${PKG_NAME}:dev From ccd665529c301290cba6d82bbb7b79fa54427ab9 Mon Sep 17 00:00:00 2001 From: CADIC Jean Maximilien Date: Tue, 6 May 2025 23:06:56 +0900 Subject: [PATCH 9/9] fix ci --- .github/workflows/trigger.yml | 15 ++++----------- Taskfile.yml | 6 ++++++ compose.yml | 4 ++-- docker/sandbox/Dockerfile | 2 +- docker/vanilla/compose.yml | 9 --------- wsd/__init__.py | 2 +- 6 files changed, 14 insertions(+), 24 deletions(-) delete mode 100644 docker/vanilla/compose.yml diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml index a7210b2..cf21ff7 100644 --- a/.github/workflows/trigger.yml +++ b/.github/workflows/trigger.yml @@ -25,12 +25,9 @@ jobs: # Install task sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin - # Install pre-commit - pip install pre-commit - pre-commit install + # Install the package + task install.package - # Build (docker + wsd[all]) - task build shell: bash build_mac: @@ -49,11 +46,7 @@ jobs: # Install task sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin - # Install pre-commit - pip install pre-commit - pre-commit install - - # Build - pip install wsd[all] --break-system-packages + # Install the package + task install.package shell: bash diff --git a/Taskfile.yml b/Taskfile.yml index a75a311..186efe3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -113,3 +113,9 @@ tasks: dataset: cmds: - gcloud auth application-default login + + install.package: + cmds: + - pip install -U pre-commit --break-system-packages # Install pre-commit + - pre-commit install + - pip install -U wsd[all] --break-system-packages # Install the package diff --git a/compose.yml b/compose.yml index 8ca7e9d..7c26ae6 100644 --- a/compose.yml +++ b/compose.yml @@ -14,7 +14,7 @@ x-mount_local: &mount_local services: sandbox: image: asia.gcr.io/linalgo/${PKG_NAME} - <<: [ *default, *mount_local ] + <<: [ *default ] build: context: $PJ_DIR dockerfile: docker/sandbox/Dockerfile @@ -25,7 +25,7 @@ services: dev: image: asia.gcr.io/linalgo/${PKG_NAME}:dev container_name: wsd-dev - <<: [ *default, *mount_local ] + <<: [ *default ] build: context: $PJ_DIR dockerfile: docker/sandbox/Dockerfile diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile index 8aa5870..b03dceb 100644 --- a/docker/sandbox/Dockerfile +++ b/docker/sandbox/Dockerfile @@ -10,6 +10,6 @@ COPY README.md ./README.md COPY wsd ./wsd # Install the library -RUN pip install .[$EXTRA] --break-system-packages && rm -r pyproject.toml README.md .env wsd +RUN pip install wsd[$EXTRA] --break-system-packages && rm -r pyproject.toml README.md .env wsd ENTRYPOINT [ "bash" ] diff --git a/docker/vanilla/compose.yml b/docker/vanilla/compose.yml deleted file mode 100644 index 967a355..0000000 --- a/docker/vanilla/compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - vanilla: - image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 - build: - context: ${PJ_DIR} - dockerfile: docker/vanilla/Dockerfile - args: - - PY_TAG=3.10-slim - target: dev diff --git a/wsd/__init__.py b/wsd/__init__.py index 1f1436a..ee57fa9 100644 --- a/wsd/__init__.py +++ b/wsd/__init__.py @@ -1,4 +1,4 @@ """Init file containing version and build""" __version__ = "0.0.1rc0" -__build__ = "Mon Apr 28 12:41:54 JST 2025" +__build__ = "Tue May 6 11:02:04 PM JST 2025"