diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b25701b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.ruff_cache diff --git a/.env.example b/.env.example index 6eb897f..029bb09 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,6 @@ LINHUB_ORG=57512466-a2eb-414b-a332-ffd5486a6fb1 LINHUB_TASK=a5040509-de9c-4757-8cb3-9087b5191a2e LINHUB_ENTITY=07a2fdef-d260-4f30-a350-8a7afc2af2a5 PKG_NAME=wsd -PJ_DIR=$PWD -COMPOSE_FILE=./docker/vanilla/compose.yml:./compose.yml FORCE_PRE_COMMIT=true -FORCE_DOWNLOAD=true +FORCE_DOWNLOAD=false +PJ_DIR=$PWD diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml index a2f8c1c..cf21ff7 100644 --- a/.github/workflows/trigger.yml +++ b/.github/workflows/trigger.yml @@ -7,7 +7,7 @@ on: branches: ["main"] jobs: - build_only: + build_linux: if: github.event_name == 'push' runs-on: ubuntu-latest outputs: @@ -25,19 +25,9 @@ jobs: # Install task sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin - # Install uv - task install_uv + # Install the package + task install.package - # Set python version - uv venv --python=3.10 - source .venv/bin/activate - - # Install pre-commit - uv pip install pre-commit - pre-commit install - - # Build - task build shell: bash build_mac: @@ -56,22 +46,7 @@ jobs: # Install task sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin - # Install uv - task install_uv - - # Set python version - uv venv --python=3.10 && source .venv/bin/activate - - # Install pre-commit - uv pip install pre-commit - pre-commit install - - # Build - task build.wheel - - # Install package - sudo mkdir /tmp/wheels - sudo cp dist/*.whl /tmp/wheels/ - sudo uv pip install $(ls /tmp/wheels | sed 's|^|/tmp/wheels/|') --system --break-system-packages + # Install the package + task install.package shell: bash diff --git a/README.md b/README.md index a0de3b4..9d58d80 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Fast, efficient, open source & open data Word Sense Disambiguation models for pr The easiest way to install `wsd` is to use pip: ``` -pip install wsd +pip install wsd[all] ``` You will also need the [The JMDict Project](https://www.edrdg.org/jmdict/j_jmdict.html) dictionary. You can use the following helper to download the file: diff --git a/Taskfile.yml b/Taskfile.yml index 7bf8b17..186efe3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,18 +9,16 @@ env: sh: grep 'version =' pyproject.toml | cut -d '"' -f 2 RSBUILD: sh: if command -v rsbuild > /dev/null; then echo "true"; else echo "false"; fi - PJ_DIR: - sh: echo $PWD JMDICT: "https://drive.usercontent.google.com/download?id=1dlvguHuMjDmtpA4beu1WaVbDte5j4SLk&confirm=yy" XLWSD: "https://drive.usercontent.google.com/download?id=1d75OUrM3dyAvYUsnBXle-Z1W0NHyKIBD&confirm=yy" VANILLA: docker run --rm - -v ${PWD}:/workspace + -v ${PJ_DIR}:/workspace --workdir /workspace --entrypoint bash -i asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 -c DEV: docker run --rm - -v ${PWD}:/workspace + -v ${PJ_DIR}:/workspace --workdir /workspace --entrypoint bash -i asia.gcr.io/linalgo/${PKG_NAME}:dev @@ -55,18 +53,6 @@ tasks: cmds: - 'echo "rsbuild is missing"' - install_uv: - desc: "Install the uv package if not already installed" - cmds: - - | - if ! which uv > /dev/null 2>&1; then - echo "Installing uv..."; - curl -LsSf https://astral.sh/uv/install.sh | sh; - sudo mv $HOME/.local/bin/uv /usr/local/bin/ - else - echo "uv is already installed."; - fi - add_build.mac: cmds: - sed -i '' 's/__version__ = ".*"/__version__ = "{{.VERSION}}"/' "${PKG_NAME}/__init__.py"; @@ -94,23 +80,13 @@ tasks: # Build Tasks build: desc: "Tasks related to building the project" - deps: [install_uv, add_build] + deps: [add_build] cmds: - if [ "{{.FORCE_DOWNLOAD}}" = "true" ]; then task download; fi - task build.docker - build.wheel: - desc: "Build the project wheel package" - deps: [install_uv, add_build] - cmds: - - if [ "{{.FORCE_PRE_COMMIT}}" = "true" ]; then pre-commit run -a; fi - - "mkdir -p dist/legacy && mv dist/*.whl dist/legacy/ || true" - - uv build --wheel - - task clean - build.docker: desc: "Build the Docker image for the project" - deps: [build.wheel] cmds: - docker compose build vanilla - docker compose build @@ -137,3 +113,9 @@ tasks: dataset: cmds: - gcloud auth application-default login + + install.package: + cmds: + - pip install -U pre-commit --break-system-packages # Install pre-commit + - pre-commit install + - pip install -U wsd[all] --break-system-packages # Install the package diff --git a/compose.yml b/compose.yml index ac4429e..7c26ae6 100644 --- a/compose.yml +++ b/compose.yml @@ -3,31 +3,40 @@ x-default: &default tty: true restart: always -x-dev: &dev +x-mount_local: &mount_local user: foo + env_file: + - .env entrypoint: bash volumes: - - ./:/workspace + - $PJ_DIR:/workspace services: sandbox: image: asia.gcr.io/linalgo/${PKG_NAME} <<: [ *default ] build: - context: ${PJ_DIR} + context: $PJ_DIR dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 + - EXTRA=all dev: image: asia.gcr.io/linalgo/${PKG_NAME}:dev container_name: wsd-dev - <<: [ *default, *dev ] + <<: [ *default ] build: - context: ${PJ_DIR} + context: $PJ_DIR dockerfile: docker/sandbox/Dockerfile args: - PY_VER=310 - EXTRA=dev - volumes: - - ${PJ_DIR}:/workspace + vanilla: + image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 + build: + context: $PJ_DIR + dockerfile: docker/vanilla/Dockerfile + args: + - PY_TAG=3.10-slim + target: dev diff --git a/docker/README.md b/docker/README.md index 179063a..ed92758 100644 --- a/docker/README.md +++ b/docker/README.md @@ -43,21 +43,6 @@ Before you begin, `deactivate your virtual environment if any` and ensure you h which task ``` -- [uv](https://docs.astral.sh/uv/getting-started/installation/) - - - Install the `uv` package if not already installed: - - ```bash - curl -LsSf https://astral.sh/uv/install.sh | sh - sudo mv $HOME/.local/bin/uv /usr/local/bin/ - ``` - - - Make sure the following command doesn't return an error - - ```bash - which uv - ``` - ## Installation To set up the project, follow these steps: @@ -75,34 +60,22 @@ To set up the project, follow these steps: cp .env.example .env ``` -3. **Create the virtual environment with uv**: - - ```bash - uv venv --python=3.10 && source .venv/bin/activate - ``` - -4. **Install [pre-commit](https://pre-commit.com/)**: +3. **Install [pre-commit](https://pre-commit.com/)**: If `pre-commit` is not already installed, you can install it using the following command: ```bash - uv pip install pre-commit - pre-commit run -a - ``` - -5. **Build the wheel**: - - ```bash - task build.wheel + pip install pre-commit + pre-commit install ``` -6. **Build docker image**: +4. **Build docker image**: ```bash - task build.docker + task build ``` -7. **Access the Docker Container**: +5. **Access the Docker Container**: ```bash docker compose up dev -d diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile index 5f1aa81..b03dceb 100644 --- a/docker/sandbox/Dockerfile +++ b/docker/sandbox/Dockerfile @@ -3,9 +3,13 @@ FROM asia.gcr.io/linalgo/wsd:vanilla-py${PY_VER} ARG EXTRA +# Copy the project files +COPY pyproject.toml ./pyproject.toml +COPY .env ./.env +COPY README.md ./README.md +COPY wsd ./wsd + # Install the library -COPY dist/*.whl /tmp/wheels/ -RUN uv pip install $(ls /tmp/wheels | sed 's|^|/tmp/wheels/|')[$EXTRA] --system && \ - sudo rm /tmp/wheels/*.whl +RUN pip install wsd[$EXTRA] --break-system-packages && rm -r pyproject.toml README.md .env wsd ENTRYPOINT [ "bash" ] diff --git a/docker/vanilla/compose.yml b/docker/vanilla/compose.yml deleted file mode 100644 index 22bc19a..0000000 --- a/docker/vanilla/compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - vanilla: - image: asia.gcr.io/linalgo/${PKG_NAME}:vanilla-py310 - build: - context: ${PJ_DIR} - dockerfile: ./docker/vanilla/Dockerfile - args: - - PY_TAG=3.10-slim - target: dev diff --git a/pyproject.toml b/pyproject.toml index a7dbd9c..447bb7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,9 @@ version = "0.0.1rc0" description = "Multi-lingual Word Sense Disambiguation." readme = "README.md" requires-python = ">=3.10" -license = { text = "MIT" } +license = "MIT" classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] authors = [ @@ -31,6 +30,17 @@ dependencies = [ [project.optional-dependencies] dev = ["pytest", "pre-commit"] +all = [ + "fugashi[unidic]", + "google-genai", + "linalgo==0.1.4", + "mesop", + "numpy", + "pillow", + "rich", + "scikit-learn", + "typer", +] [project.urls] Homepage = "https://github.com/linalgo/wsd" diff --git a/wsd/__init__.py b/wsd/__init__.py index 073ea04..ee57fa9 100644 --- a/wsd/__init__.py +++ b/wsd/__init__.py @@ -1,4 +1,4 @@ """Init file containing version and build""" __version__ = "0.0.1rc0" -__build__ = "Wed Mar 26 20:57:54 JST 2025" +__build__ = "Tue May 6 11:02:04 PM JST 2025"