From 997b997d477bbe46caa000906b67f989dd2ebeb0 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 15:38:18 -0700 Subject: [PATCH 01/68] manylinux wheel --- .github/workflows/manylinux_wheel | 40 +++++++++++++++++++ .../manylinux_wheel/1_prerequisites.sh | 8 ++++ .../scripts_new/manylinux_wheel/2_build.sh | 5 +++ .../scripts_new/manylinux_wheel/3_check.sh | 6 +++ .../manylinux_wheel/4_install_wheel.sh | 12 ++++++ .../scripts_new/manylinux_wheel/5_test.sh | 8 ++++ 6 files changed, 79 insertions(+) create mode 100644 .github/workflows/manylinux_wheel create mode 100644 .github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh create mode 100644 .github/workflows/scripts_new/manylinux_wheel/2_build.sh create mode 100644 .github/workflows/scripts_new/manylinux_wheel/3_check.sh create mode 100644 .github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh create mode 100644 .github/workflows/scripts_new/manylinux_wheel/5_test.sh diff --git a/.github/workflows/manylinux_wheel b/.github/workflows/manylinux_wheel new file mode 100644 index 0000000000..a6d76ba3da --- /dev/null +++ b/.github/workflows/manylinux_wheel @@ -0,0 +1,40 @@ +name: Manylinux wheel build +on: + pull_request: + branches: + - main + workflow_dispatch: +jobs: + build_wheel: + name: Manylinux wheel Build + runs-on: ubuntu-22.04 + container: quay.io/pypa/manylinux_2_28_x86_64:latest + steps: + - uses: actions/checkout@v4 + - name: manylinux wheel prerequisites + run: | + bash .github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh + - name: manylinux wheel build + run: | + bash .github/workflows/scripts_new/manylinux_wheel/2_build.sh + - name: manylinux wheel check + run: | + bash .github/workflows/scripts_new/manylinux_wheel/3_check.sh + - uses: actions/upload-artifact@v4 + with: + name: manylinux_wheel + path: dist/*.whl + test_wheel: + name: Manylinux wheel Test + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: manylinux_wheel + - name: manylinux install wheel + run: | + bash .github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh + - name: manylinux test + run: | + bash .github/workflows/scripts_new/manylinux_wheel/5_test.sh diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh new file mode 100644 index 0000000000..b2a6b14c3e --- /dev/null +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +yum update +yum install -y git clang libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel +git submodule update --init --jobs 2 + diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh new file mode 100644 index 0000000000..fc4aa23653 --- /dev/null +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -ex + +./build.py wheel diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh new file mode 100644 index 0000000000..092a8acd2c --- /dev/null +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -ex + +~/.cache/ti-build-cache/mambaforge/envs/3.12/bin/pip install auditwheel +auditwheel show dist/*.whl diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh new file mode 100644 index 0000000000..b544e1d6a2 --- /dev/null +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -ex + +ls -lR manylinux_wheel + +python -V +pip -V + +pip install manylinux_wheel/dist/*.whl + +python -c 'import taichi as ti; ti.init(arch=ti.cpu); print(ti.__version__)' diff --git a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh new file mode 100644 index 0000000000..698549c632 --- /dev/null +++ b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +ls -lR + +pip install -r requirements_test.txt +python tests/run_tests.py From ca93ffebce1ae6a1b15e9f2c95039535228b127a Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 15:39:16 -0700 Subject: [PATCH 02/68] .yml --- .github/workflows/{manylinux_wheel => manylinux_wheel.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{manylinux_wheel => manylinux_wheel.yml} (100%) diff --git a/.github/workflows/manylinux_wheel b/.github/workflows/manylinux_wheel.yml similarity index 100% rename from .github/workflows/manylinux_wheel rename to .github/workflows/manylinux_wheel.yml From 14843ec51e557c15799b9ff1f8378cca1e30b41a Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 15:39:40 -0700 Subject: [PATCH 03/68] needs --- .github/workflows/manylinux_wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index a6d76ba3da..2e456c3dc8 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -27,6 +27,7 @@ jobs: test_wheel: name: Manylinux wheel Test runs-on: ubuntu-22.04 + needs: build_wheel steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 From 22b78be6f0bb36b8223bfd5f2a03bdd2f5740631 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 15:44:29 -0700 Subject: [PATCH 04/68] make w safe --- .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index b2a6b14c3e..fcfe568bfc 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -4,5 +4,5 @@ set -ex yum update yum install -y git clang libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel +git config --global --add safe.directory /__w/taichi/taichi git submodule update --init --jobs 2 - From d599e935515860eb3f58785a51fa6f5e8b9bdbce Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 15:45:20 -0700 Subject: [PATCH 05/68] precommit --- .github/workflows/scripts_new/manylinux_wheel/5_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh index 698549c632..27328f3417 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh @@ -2,7 +2,7 @@ set -ex -ls -lR +ls -lR pip install -r requirements_test.txt python tests/run_tests.py From 2e97b77c22ad098ce2601eba4862d0377b06e5c8 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 18:35:29 -0700 Subject: [PATCH 06/68] more exploration --- .../workflows/scripts_new/manylinux_wheel/4_install_wheel.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh index b544e1d6a2..940a595f1b 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -2,7 +2,10 @@ set -ex -ls -lR manylinux_wheel +ls -l +find . -cmin -1 +find . -cmin -1 -name '*.whl +find . -name '*.whl python -V pip -V From 7eb1cb7d3ea1371998102c33043f3fe54c8220a4 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 17 Jun 2025 20:07:35 -0700 Subject: [PATCH 07/68] try different wheel path --- .../workflows/scripts_new/manylinux_wheel/4_install_wheel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh index 940a595f1b..c848263d11 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -6,10 +6,11 @@ ls -l find . -cmin -1 find . -cmin -1 -name '*.whl find . -name '*.whl +ls -lh *.whl python -V pip -V -pip install manylinux_wheel/dist/*.whl +pip install *.whl python -c 'import taichi as ti; ti.init(arch=ti.cpu); print(ti.__version__)' From 647763d1e75b678fd150792477a3f0a1db5d0888 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 16:17:05 -0700 Subject: [PATCH 08/68] save stuff I"m trying --- .github/workflows/manylinux_wheel.yml | 16 ++++++++++++++-- .../scripts_new/manylinux_wheel/2_build.sh | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 2e456c3dc8..8a6c21ea73 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -9,6 +9,10 @@ jobs: name: Manylinux wheel Build runs-on: ubuntu-22.04 container: quay.io/pypa/manylinux_2_28_x86_64:latest + strategy: + matrix: + PYTHON_VERSION: ['cp310'] + # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - name: manylinux wheel prerequisites @@ -16,23 +20,31 @@ jobs: bash .github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh - name: manylinux wheel build run: | + export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH + python -V bash .github/workflows/scripts_new/manylinux_wheel/2_build.sh - name: manylinux wheel check run: | + export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH + python -V bash .github/workflows/scripts_new/manylinux_wheel/3_check.sh - uses: actions/upload-artifact@v4 with: - name: manylinux_wheel + name: manylinux_wheel_${{ matrix.PYTHON_VERSION }} path: dist/*.whl test_wheel: name: Manylinux wheel Test runs-on: ubuntu-22.04 needs: build_wheel + strategy: + matrix: + PYTHON_VERSION: ['cp310'] + # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: manylinux_wheel + name: manylinux_wheel_${{ matrix.PYTHON_VERSION }} - name: manylinux install wheel run: | bash .github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index fc4aa23653..2fa13a5e13 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -2,4 +2,5 @@ set -ex +export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH ./build.py wheel From c4c4b7d95a108ac28a4768bbc36f2c3832fa3235 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 19:15:30 -0700 Subject: [PATCH 09/68] setting python version --- .github/workflows/manylinux_wheel.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 8a6c21ea73..0640b7f7f9 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -38,13 +38,19 @@ jobs: needs: build_wheel strategy: matrix: - PYTHON_VERSION: ['cp310'] + include: + - PYTHON_CP_VERSION: 'cp310' + PYTHON_VERSION: '3.10' # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: manylinux_wheel_${{ matrix.PYTHON_VERSION }} + name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} + - name: Python check + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.PYTHON_VERSION }} - name: manylinux install wheel run: | bash .github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh From e6a2ca7274146d132bb6fab046968fad6b2cdf8c Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 19:20:08 -0700 Subject: [PATCH 10/68] udpate python path --- .github/workflows/manylinux_wheel.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 0640b7f7f9..5d54a80dd9 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -11,8 +11,8 @@ jobs: container: quay.io/pypa/manylinux_2_28_x86_64:latest strategy: matrix: - PYTHON_VERSION: ['cp310'] - # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] + PYTHON_CP_VERSION: ['cp310'] + # PYTHON_CP_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - name: manylinux wheel prerequisites @@ -20,17 +20,17 @@ jobs: bash .github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh - name: manylinux wheel build run: | - export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH + export PATH=/opt/python/${{ matrix.PYTHON_CP_VERSION }}/bin:$PATH python -V bash .github/workflows/scripts_new/manylinux_wheel/2_build.sh - name: manylinux wheel check run: | - export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH + export PATH=/opt/python/${{ matrix.PYTHON_CP_VERSION }}/bin:$PATH python -V bash .github/workflows/scripts_new/manylinux_wheel/3_check.sh - uses: actions/upload-artifact@v4 with: - name: manylinux_wheel_${{ matrix.PYTHON_VERSION }} + name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} path: dist/*.whl test_wheel: name: Manylinux wheel Test From 6f2a6e4e5d34f00cc8ea0b43651260bb7bc0a59b Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 19:30:06 -0700 Subject: [PATCH 11/68] fix path --- .github/workflows/manylinux_wheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 5d54a80dd9..a3a21e808f 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -20,12 +20,12 @@ jobs: bash .github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh - name: manylinux wheel build run: | - export PATH=/opt/python/${{ matrix.PYTHON_CP_VERSION }}/bin:$PATH + export PATH=/opt/python/${{ matrix.PYTHON_CP_VERSION }}-${{ matrix.PYTHON_CP_VERSION }}/bin:$PATH python -V bash .github/workflows/scripts_new/manylinux_wheel/2_build.sh - name: manylinux wheel check run: | - export PATH=/opt/python/${{ matrix.PYTHON_CP_VERSION }}/bin:$PATH + export PATH=/opt/python/${{ matrix.PYTHON_CP_VERSION }}-${{ matrix.PYTHON_CP_VERSION }}/bin:$PATH python -V bash .github/workflows/scripts_new/manylinux_wheel/3_check.sh - uses: actions/upload-artifact@v4 From e60d4a3531926f2e97cbcf2ea065a54f41b82109 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 20:02:38 -0700 Subject: [PATCH 12/68] api_doc.html --- .github/workflows/api_doc.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/api_doc.yml diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml new file mode 100644 index 0000000000..873ca887c4 --- /dev/null +++ b/.github/workflows/api_doc.yml @@ -0,0 +1,30 @@ +name: api docs build + +on: + push: + branches: [ "main" ] + workflow_dispatch: + +concurrency: + group: gh-pages-deploy-${{ github.ref }} + cancel-in-progress: true +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build + run: | + pip install pdoc + pdoc --output-dir public taichi + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./public + destination_dir: ${{ github.ref_name }} + keep_files: true + publish_branch: gh-pages From 52b141f19eb5c90ba834309d5ce52f78fe65ee77 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 20:30:53 -0700 Subject: [PATCH 13/68] head_ref --- .github/workflows/api_doc.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index 873ca887c4..c54921c5dd 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -1,8 +1,9 @@ name: api docs build on: - push: - branches: [ "main" ] + pull_request: + branches: + - main workflow_dispatch: concurrency: @@ -25,6 +26,6 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./public - destination_dir: ${{ github.ref_name }} + destination_dir: ${{ github.head_ref }} keep_files: true publish_branch: gh-pages From 4bddab7bd6fd5af0218d77de7be52930962dd74f Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 20:37:05 -0700 Subject: [PATCH 14/68] maybe fix python path --- .github/workflows/scripts_new/manylinux_wheel/2_build.sh | 2 +- .github/workflows/scripts_new/manylinux_wheel/3_check.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index 2fa13a5e13..d9e76e02f9 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -2,5 +2,5 @@ set -ex -export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH +# export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH ./build.py wheel diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh index 092a8acd2c..9014d2fdbc 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -2,5 +2,5 @@ set -ex -~/.cache/ti-build-cache/mambaforge/envs/3.12/bin/pip install auditwheel +pip install auditwheel auditwheel show dist/*.whl From e6220bc4a7676ffbbafd5deb2e78a10f56a00074 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 24 Jun 2025 22:14:47 -0700 Subject: [PATCH 15/68] run repair --- .github/workflows/scripts_new/manylinux_wheel/3_check.sh | 2 ++ .../workflows/scripts_new/manylinux_wheel/4_install_wheel.sh | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh index 9014d2fdbc..82ed5aefcc 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -4,3 +4,5 @@ set -ex pip install auditwheel auditwheel show dist/*.whl +auditwheel repair -v dist/*.whl +auditwheel show dist/*.whl diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh index c848263d11..6de08ea8ec 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -2,8 +2,6 @@ set -ex -ls -l -find . -cmin -1 find . -cmin -1 -name '*.whl find . -name '*.whl ls -lh *.whl From 4548d0ec8a6f3c9c520ba665f66d90da76e167c3 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 11:02:57 -0700 Subject: [PATCH 16/68] remove -v --- .github/workflows/scripts_new/manylinux_wheel/3_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh index 82ed5aefcc..f46a856bf7 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -4,5 +4,5 @@ set -ex pip install auditwheel auditwheel show dist/*.whl -auditwheel repair -v dist/*.whl +auditwheel repair dist/*.whl auditwheel show dist/*.whl From 4d4ea8146bdc0317720167717271fd02cdee1663 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 11:07:31 -0700 Subject: [PATCH 17/68] remove yum update --- .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index fcfe568bfc..50f9f9cd7a 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -2,7 +2,7 @@ set -ex -yum update +# yum update yum install -y git clang libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel git config --global --add safe.directory /__w/taichi/taichi git submodule update --init --jobs 2 From 6199156723b8aa3267bf9fa489d4832638e6be6b Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 15:36:35 -0700 Subject: [PATCH 18/68] push to s3 --- .github/workflows/manylinux_wheel.yml | 51 +++++++++++++++++-- .../manylinux_wheel/4_install_wheel.sh | 2 +- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index a3a21e808f..72a9253f01 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -32,10 +32,40 @@ jobs: with: name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} path: dist/*.whl + + upload_to_storaage: + name: Manylinux wheel upload + runs-on: ubuntu-22.04 + needs: build_wheel + environment: aws_storage + strategy: + matrix: + PYTHON_CP_VERSION: ['cp310'] + steps: + - uses: actions/checkout@v4 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - uses: actions/download-artifact@v4 + with: + name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} + - name: 'Upload wheel to aws s3 storage' + run: | + set -x + ls -l *.whl + mkdir -p dist + mv *.whl dist/ + aws s3 ls + aws s3 ls s3://genesis-ai-compiler/ + aws s3 sync dist/ s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ test_wheel: name: Manylinux wheel Test runs-on: ubuntu-22.04 - needs: build_wheel + needs: upload_to_storaage + environment: aws_storage strategy: matrix: include: @@ -44,15 +74,30 @@ jobs: # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + # - uses: actions/download-artifact@v4 + # with: + # name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Download artifact + run: | + set -x + aws s3 ls s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/ + aws s3 ls s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ + aws s3 sync s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_VERSION }}/ dist/ - name: Python check uses: actions/setup-python@v4 with: python-version: ${{ matrix.PYTHON_VERSION }} - name: manylinux install wheel run: | + set -x + ls -l + ls -l dist/ bash .github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh - name: manylinux test run: | diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh index 6de08ea8ec..89afbac8cd 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -9,6 +9,6 @@ ls -lh *.whl python -V pip -V -pip install *.whl +pip install dist/*.whl python -c 'import taichi as ti; ti.init(arch=ti.cpu); print(ti.__version__)' From 4c0e753da025df21da82e25b61b69a7e107b2a1a Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 19:57:46 -0700 Subject: [PATCH 19/68] find both clang and clang++ --- .github/workflows/scripts/ti_build/compiler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts/ti_build/compiler.py b/.github/workflows/scripts/ti_build/compiler.py index a6befa0f06..db56a11dcf 100644 --- a/.github/workflows/scripts/ti_build/compiler.py +++ b/.github/workflows/scripts/ti_build/compiler.py @@ -35,9 +35,8 @@ def setup_clang(as_compiler=True) -> None: if u.system == "Linux": for v in ("-15", "-14", "-13", "-12", "-11", "-10", ""): clang = shutil.which(f"clang{v}") - if clang is not None: - clangpp = shutil.which(f"clang++{v}") - assert clangpp + clangpp = shutil.which(f"clang++{v}") + if clang is not None and clangpp is not None: break else: error("Could not find clang of any version") From e981e17c5b69d6ab2ba1aaea030ebb488c7d2ea8 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 21:01:20 -0700 Subject: [PATCH 20/68] download clang; skip compiler checks --- .github/workflows/scripts/ti_build/compiler.py | 1 + .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 5 ++++- .github/workflows/scripts_new/manylinux_wheel/2_build.sh | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/ti_build/compiler.py b/.github/workflows/scripts/ti_build/compiler.py index db56a11dcf..86940e9eb6 100644 --- a/.github/workflows/scripts/ti_build/compiler.py +++ b/.github/workflows/scripts/ti_build/compiler.py @@ -58,6 +58,7 @@ def setup_clang(as_compiler=True) -> None: raise RuntimeError(f"Unsupported platform: {u.system} {u.machine}") cmake_args["CLANG_EXECUTABLE"] = clang + cmake_args["CMAKE_CXX_COMPILER_WORKS"] = "1" if as_compiler: cc = os.environ.get("CC") diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index 50f9f9cd7a..e596f2df11 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -3,6 +3,9 @@ set -ex # yum update -yum install -y git clang libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel +yum install -y git libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel git config --global --add safe.directory /__w/taichi/taichi git submodule update --init --jobs 2 + +wget https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.4/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz +tar -xf clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index d9e76e02f9..0134d782af 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -3,4 +3,6 @@ set -ex # export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH +export PATH=$PWD/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4/bin:$PATH + ./build.py wheel From 1b54af0ce3e82f8509a7151d23dfc70b95a8f1b1 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 21:03:52 -0700 Subject: [PATCH 21/68] install wget --- .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index e596f2df11..2038503243 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -3,7 +3,7 @@ set -ex # yum update -yum install -y git libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel +yum install -y git wget libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel git config --global --add safe.directory /__w/taichi/taichi git submodule update --init --jobs 2 From f6828a705233930bfda69709b4af6432a89220fb Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 26 Jun 2025 21:18:00 -0700 Subject: [PATCH 22/68] -q --- .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index 2038503243..e86f3cad19 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -7,5 +7,5 @@ yum install -y git wget libXrandr-devel libXinerama-devel libXcursor-devel libXi git config --global --add safe.directory /__w/taichi/taichi git submodule update --init --jobs 2 -wget https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.4/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz +wget -q https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.4/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz tar -xf clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz From 358f85b5e92ca098030af993748a758226182668 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 10:38:25 -0700 Subject: [PATCH 23/68] make apple defintiosn conditional on apple --- CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5113468159..1d29d14d70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,12 +72,13 @@ option(TI_WITH_BACKTRACE "Use backward-cpp to print out C++ stack trace upon fai option(TI_GENERATE_PDB "Generate Program Database (PDB) files (will make compilation uncacheable)" OFF) option(TI_WITH_LTO "Enable Link Time Optimization (LTO) (affects Windows + MSVC for now)" OFF) # wheel-tag: lto -if(LINUX OR APPLE) - if (NOT IOS) - # (penguinliong) Not compatible with -fembed-bitcode. Not sure where it - # comes from; probably a XCode preset? - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") - endif() +if((LINUX OR APPLE) AND NOT IOS) + # (penguinliong) Not compatible with -fembed-bitcode. Not sure where it + # comes from; probably a XCode preset? + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections") +endif() + +if(APPLE) add_definitions(-D__MAC_OS_X_VERSION_MIN_REQUIRED=140000) add_definitions(-DTARGET_OS_OSX) endif() From 85b7a6188dd3fb2577d03b5bc1de8e4426c2c33d Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 12:06:15 -0700 Subject: [PATCH 24/68] always set the -Wno flag --- cmake/TaichiCXXFlags.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/TaichiCXXFlags.cmake b/cmake/TaichiCXXFlags.cmake index 9f87bddb51..34a54e0db1 100644 --- a/cmake/TaichiCXXFlags.cmake +++ b/cmake/TaichiCXXFlags.cmake @@ -101,10 +101,10 @@ else() # FIXME: Check why Android don't support check_cxx_compiler_flag if (NOT ANDROID) - check_cxx_compiler_flag("-Wno-unused-but-set-variable" CXX_HAS_Wno_unused_but_set_variable) - if (${CXX_HAS_Wno_unused_but_set_variable}) + # if we don't set -Wno-unused-but-set-variable, then Eigen build will fail anyway, + # in a way that is not obvious why, without very close inspection. + # So we will simply set it set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable ") - endif() endif() endif () From f558502ec8d7d9f102c4d793926a92382aad6f70 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 12:06:32 -0700 Subject: [PATCH 25/68] link libstdc++.so, so can be found easily inchecks --- .../scripts_new/manylinux_wheel/1_prerequisites.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index e86f3cad19..f2229b9840 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -9,3 +9,11 @@ git submodule update --init --jobs 2 wget -q https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.4/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz tar -xf clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz + +# clang++ searches for libstd++.so, not libstdc++.so.6 +# without this, then the compiler checks will fail +# eg: +# - check for working compiler itself +# - and also check for -Wno-unused-but-set-variable, in TaichiCXXFlags.cmake +# which will cause obscure compile errors for external/Eigen +ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so From 11566454285240bd080bd8a07c4da9f28c08d2c9 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 13:39:14 -0700 Subject: [PATCH 26/68] install static libstdc++ lib --- .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index f2229b9840..0f26aff490 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -17,3 +17,7 @@ tar -xf clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4.tar.xz # - and also check for -Wno-unused-but-set-variable, in TaichiCXXFlags.cmake # which will cause obscure compile errors for external/Eigen ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so + +# since we are linking statically +# and looks like this installs the same version of libstdc++-static as libstdc++ +yum install libstdc++-static From cecaffd2128daa166439623ca5ff03afe8026783 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 13:41:53 -0700 Subject: [PATCH 27/68] remove android thing, since probalby old, and eigen wont compile anyway without it --- cmake/TaichiCXXFlags.cmake | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cmake/TaichiCXXFlags.cmake b/cmake/TaichiCXXFlags.cmake index 34a54e0db1..333cfcc788 100644 --- a/cmake/TaichiCXXFlags.cmake +++ b/cmake/TaichiCXXFlags.cmake @@ -99,13 +99,10 @@ else() # However, some of these "constexpr" are debug flags and will be manually enabled upon debugging. set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unneeded-internal-declaration ") - # FIXME: Check why Android don't support check_cxx_compiler_flag - if (NOT ANDROID) - # if we don't set -Wno-unused-but-set-variable, then Eigen build will fail anyway, - # in a way that is not obvious why, without very close inspection. - # So we will simply set it - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable ") - endif() + # if we don't set -Wno-unused-but-set-variable, then Eigen build will fail anyway, + # in a way that is not obvious why, without very close inspection. + # So we will simply set it + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable ") endif () # (penguinliong) When building for iOS with Xcode `CMAKE_SYSTEM_PROCESSOR` From f79901dfdcd4c51963b59590446311d9010052b9 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 13:42:19 -0700 Subject: [PATCH 28/68] remove dead path line --- .github/workflows/scripts_new/manylinux_wheel/2_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index 0134d782af..3e2933fc0b 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -2,7 +2,6 @@ set -ex -# export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH export PATH=$PWD/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4/bin:$PATH ./build.py wheel From 846f6a77cbe0ba5a36e2c3ae89e2200400e07d2a Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 13:45:15 -0700 Subject: [PATCH 29/68] -y --- .../workflows/scripts_new/manylinux_wheel/1_prerequisites.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh index 0f26aff490..73d0690441 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/1_prerequisites.sh @@ -20,4 +20,4 @@ ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so # since we are linking statically # and looks like this installs the same version of libstdc++-static as libstdc++ -yum install libstdc++-static +yum install -y libstdc++-static From 69b69f8a932d0b016f5163775de44e293f41fab6 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 13:56:11 -0700 Subject: [PATCH 30/68] /wheelhouse --- .github/workflows/manylinux_wheel.yml | 2 +- .github/workflows/scripts_new/manylinux_wheel/3_check.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 72a9253f01..03c36ba388 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} - path: dist/*.whl + path: /wheelhouse/*.whl upload_to_storaage: name: Manylinux wheel upload diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh index f46a856bf7..dd0794936d 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -5,4 +5,5 @@ set -ex pip install auditwheel auditwheel show dist/*.whl auditwheel repair dist/*.whl -auditwheel show dist/*.whl +ls -lh /wheelhouse/ +auditwheel show /wheelhouse/*.whl From 5b0f4488dbcb9f4de78cd9d897c1100c57cf3afb Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 14:24:59 -0700 Subject: [PATCH 31/68] lst dist/*.whl --- .../workflows/scripts_new/manylinux_wheel/4_install_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh index 89afbac8cd..f6367eaffe 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -4,7 +4,7 @@ set -ex find . -cmin -1 -name '*.whl find . -name '*.whl -ls -lh *.whl +ls -lh dist/*.whl python -V pip -V From 2555a2bf7f5118e42f4f75172af6f6eb2d9433bc Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 14:45:29 -0700 Subject: [PATCH 32/68] hunt fo the wheel --- .github/workflows/scripts_new/manylinux_wheel/3_check.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh index dd0794936d..b2bc40d608 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -5,5 +5,8 @@ set -ex pip install auditwheel auditwheel show dist/*.whl auditwheel repair dist/*.whl -ls -lh /wheelhouse/ -auditwheel show /wheelhouse/*.whl +pwd +ls +find . -name '*.whl' +ls -lh wheelhouse/ +auditwheel show wheelhouse/*.whl From 894f6a21fa4567301f3145b770ae9400538bf49b Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 15:27:41 -0700 Subject: [PATCH 33/68] upload from wheelhouse --- .github/workflows/manylinux_wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 03c36ba388..61209f9430 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} - path: /wheelhouse/*.whl + path: wheelhouse/*.whl upload_to_storaage: name: Manylinux wheel upload From 0a0e712c3f048a9d06115dfed1d4ad20041f7c49 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 16:00:34 -0700 Subject: [PATCH 34/68] try not using special llvm --- .github/workflows/scripts/ti_build/llvm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/ti_build/llvm.py b/.github/workflows/scripts/ti_build/llvm.py index e487a560b4..51a9975475 100644 --- a/.github/workflows/scripts/ti_build/llvm.py +++ b/.github/workflows/scripts/ti_build/llvm.py @@ -23,10 +23,10 @@ def setup_llvm() -> None: if cmake_args.get_effective("TI_WITH_AMDGPU"): out = get_cache_home() / "llvm15-amdgpu-005" url = "https://github.com/GaleSeLee/assets/releases/download/v0.0.5/taichi-llvm-15.0.0-linux.zip" - elif is_manylinux2014(): - # FIXME: prebuilt llvm15 on ubuntu didn't work on manylinux2014 image of centos. Once that's fixed, remove this hack. - out = get_cache_home() / "llvm15-manylinux2014" - url = "https://github.com/ailzhang/torchhub_example/releases/download/0.3/taichi-llvm-15-linux.zip" + # elif is_manylinux2014(): + # # FIXME: prebuilt llvm15 on ubuntu didn't work on manylinux2014 image of centos. Once that's fixed, remove this hack. + # out = get_cache_home() / "llvm15-manylinux2014" + # url = "https://github.com/ailzhang/torchhub_example/releases/download/0.3/taichi-llvm-15-linux.zip" else: out = get_cache_home() / "llvm15" url = "https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/taichi-llvm-15-linux.zip" From 583d90b5db3a617b7176432abd46f792d71a5d02 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 16:48:13 -0700 Subject: [PATCH 35/68] fix sync command --- .github/workflows/manylinux_wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 61209f9430..a440aa9fff 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -88,7 +88,7 @@ jobs: set -x aws s3 ls s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/ aws s3 ls s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ - aws s3 sync s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_VERSION }}/ dist/ + aws s3 sync s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ dist/ - name: Python check uses: actions/setup-python@v4 with: From 80124b74558c61c0ef76f27d8be730dc6cf6bd88 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 17:17:47 -0700 Subject: [PATCH 36/68] addd stdc++fs --- cmake/TaichiCore.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/TaichiCore.cmake b/cmake/TaichiCore.cmake index b716b071d4..e15f0c1d79 100644 --- a/cmake/TaichiCore.cmake +++ b/cmake/TaichiCore.cmake @@ -370,6 +370,7 @@ if(TI_WITH_PYTHON) # Excluding Android # Android defaults to static linking with libc++, no tinkering needed. target_link_options(${CORE_WITH_PYBIND_LIBRARY_NAME} PUBLIC -static-libgcc -static-libstdc++) + target_link_libraries(${CORE_WITH_PYBIND_LIBRARY_NAME} PUBLIC stdc++fs) endif() endif() From 6b291ae11535cf35afaf50b0dd326fe7c0d6f30a Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 17:19:33 -0700 Subject: [PATCH 37/68] remove dead code --- .github/workflows/scripts/ti_build/llvm.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/scripts/ti_build/llvm.py b/.github/workflows/scripts/ti_build/llvm.py index 51a9975475..49e1413388 100644 --- a/.github/workflows/scripts/ti_build/llvm.py +++ b/.github/workflows/scripts/ti_build/llvm.py @@ -23,10 +23,6 @@ def setup_llvm() -> None: if cmake_args.get_effective("TI_WITH_AMDGPU"): out = get_cache_home() / "llvm15-amdgpu-005" url = "https://github.com/GaleSeLee/assets/releases/download/v0.0.5/taichi-llvm-15.0.0-linux.zip" - # elif is_manylinux2014(): - # # FIXME: prebuilt llvm15 on ubuntu didn't work on manylinux2014 image of centos. Once that's fixed, remove this hack. - # out = get_cache_home() / "llvm15-manylinux2014" - # url = "https://github.com/ailzhang/torchhub_example/releases/download/0.3/taichi-llvm-15-linux.zip" else: out = get_cache_home() / "llvm15" url = "https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/taichi-llvm-15-linux.zip" From 82ca8223ddd4f5e4b5a4545824ca82fe2463bd20 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 17:20:46 -0700 Subject: [PATCH 38/68] remove more dead code --- .github/workflows/manylinux_wheel.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index a440aa9fff..9b0230f131 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -74,9 +74,6 @@ jobs: # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - # - uses: actions/download-artifact@v4 - # with: - # name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: From 58587bd4e414164f0cffbbfb2f2677381d1a64d3 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 18:03:07 -0700 Subject: [PATCH 39/68] remove some scaffolding --- .github/workflows/manylinux_wheel.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 9b0230f131..13a5c0f7e8 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -58,8 +58,6 @@ jobs: ls -l *.whl mkdir -p dist mv *.whl dist/ - aws s3 ls - aws s3 ls s3://genesis-ai-compiler/ aws s3 sync dist/ s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ test_wheel: name: Manylinux wheel Test @@ -83,8 +81,6 @@ jobs: - name: Download artifact run: | set -x - aws s3 ls s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/ - aws s3 ls s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ aws s3 sync s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/${{ matrix.PYTHON_CP_VERSION }}/ dist/ - name: Python check uses: actions/setup-python@v4 @@ -93,7 +89,6 @@ jobs: - name: manylinux install wheel run: | set -x - ls -l ls -l dist/ bash .github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh - name: manylinux test From 45033809489ca0fc2ad36d44643039fe6c7f16e5 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 18:03:17 -0700 Subject: [PATCH 40/68] run py 3.10 -> 3.13 --- .github/workflows/manylinux_wheel.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 13a5c0f7e8..25dcbf99ce 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -11,8 +11,7 @@ jobs: container: quay.io/pypa/manylinux_2_28_x86_64:latest strategy: matrix: - PYTHON_CP_VERSION: ['cp310'] - # PYTHON_CP_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] + PYTHON_CP_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - name: manylinux wheel prerequisites @@ -40,7 +39,7 @@ jobs: environment: aws_storage strategy: matrix: - PYTHON_CP_VERSION: ['cp310'] + PYTHON_CP_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] steps: - uses: actions/checkout@v4 - name: Configure AWS credentials @@ -69,7 +68,12 @@ jobs: include: - PYTHON_CP_VERSION: 'cp310' PYTHON_VERSION: '3.10' - # PYTHON_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] + - PYTHON_CP_VERSION: 'cp311' + PYTHON_VERSION: '3.11' + - PYTHON_CP_VERSION: 'cp312' + PYTHON_VERSION: '3.12' + - PYTHON_CP_VERSION: 'cp313' + PYTHON_VERSION: '3.13' steps: - uses: actions/checkout@v4 - name: Configure AWS credentials From f167a92c0aefc2838ec59b78dbfce1e411b2715c Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 18:05:18 -0700 Subject: [PATCH 41/68] remove some more spam --- .github/workflows/scripts_new/manylinux_wheel/3_check.sh | 2 -- .../scripts_new/manylinux_wheel/4_install_wheel.sh | 8 -------- .github/workflows/scripts_new/manylinux_wheel/5_test.sh | 2 -- 3 files changed, 12 deletions(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh index b2bc40d608..3cd4ce586d 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/3_check.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/3_check.sh @@ -6,7 +6,5 @@ pip install auditwheel auditwheel show dist/*.whl auditwheel repair dist/*.whl pwd -ls -find . -name '*.whl' ls -lh wheelhouse/ auditwheel show wheelhouse/*.whl diff --git a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh index f6367eaffe..1a29a3a3e9 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/4_install_wheel.sh @@ -2,13 +2,5 @@ set -ex -find . -cmin -1 -name '*.whl -find . -name '*.whl -ls -lh dist/*.whl - -python -V -pip -V - pip install dist/*.whl - python -c 'import taichi as ti; ti.init(arch=ti.cpu); print(ti.__version__)' diff --git a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh index 27328f3417..27065227ca 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh @@ -2,7 +2,5 @@ set -ex -ls -lR - pip install -r requirements_test.txt python tests/run_tests.py From d5e2843f10246d73b626b41aedeb4594acfb28c6 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 18:45:12 -0700 Subject: [PATCH 42/68] fail-fast: false --- .github/workflows/manylinux_wheel.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 25dcbf99ce..512a2c5056 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -12,6 +12,7 @@ jobs: strategy: matrix: PYTHON_CP_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] + fail-fast: false steps: - uses: actions/checkout@v4 - name: manylinux wheel prerequisites @@ -40,6 +41,7 @@ jobs: strategy: matrix: PYTHON_CP_VERSION: ['cp310', 'cp311', 'cp312', 'cp313'] + fail-fast: false steps: - uses: actions/checkout@v4 - name: Configure AWS credentials @@ -74,6 +76,7 @@ jobs: PYTHON_VERSION: '3.12' - PYTHON_CP_VERSION: 'cp313' PYTHON_VERSION: '3.13' + fail-fast: false steps: - uses: actions/checkout@v4 - name: Configure AWS credentials From 1582b219c8c61473545797c3067f9b8a5989ab88 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 18:53:36 -0700 Subject: [PATCH 43/68] add setuptools and cffi to install requires --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index a655a09f1d..734d49876b 100644 --- a/setup.py +++ b/setup.py @@ -228,6 +228,8 @@ def sign_development_for_apple_m1(): "colorama", "dill", "rich", + "setuptools>=68.0.0", # Required for Python 3.12+ compatibility + "cffi>=1.16.0", ], data_files=[ (os.path.join("_lib", "runtime"), data_files), From 8f388592188e608aaa2289b0fcf69d10a47688ce Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 21:28:17 -0700 Subject: [PATCH 44/68] add seutptools and cffi to pyproject.toml --- pyproject.toml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a351502684..7c577c396b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,14 @@ [build-system] requires = [ - "setuptools", "wheel", - "numpy", "pybind11", "cmake", - "scikit-build", "ninja; platform_system != 'Windows'", + "setuptools", + "wheel", + "numpy", + "pybind11", + "cmake", + "scikit-build", + "ninja; platform_system != 'Windows'", + "setuptools>=68.0.0", + "cffi>=1.16.0", ] build-backend = "setuptools.build_meta" From 823fa2ab6ce0d84f8b5485ef3d57dd32e40f03fe Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sat, 28 Jun 2025 22:55:03 -0700 Subject: [PATCH 45/68] remove wget --- .../real_func/rendering/taichi_ngp.py | 1051 ----------------- requirements_test.txt | 1 - 2 files changed, 1052 deletions(-) delete mode 100644 python/taichi/examples/real_func/rendering/taichi_ngp.py diff --git a/python/taichi/examples/real_func/rendering/taichi_ngp.py b/python/taichi/examples/real_func/rendering/taichi_ngp.py deleted file mode 100644 index 720997f39f..0000000000 --- a/python/taichi/examples/real_func/rendering/taichi_ngp.py +++ /dev/null @@ -1,1051 +0,0 @@ -# type: ignore - -# Instant-NGP renderer implemented using Taichi -# Author : Linyou (linyoutian.loyot@gmail.com) -# Original code at https://github.com/Linyou/taichi-ngp-renderer - -# GUI instruction: -# 1. Move the mouse with right click to rotate the camera -# 2. Press 'w', 'a', 's', 'd' key to move the camera like FPS game -# 3. Press 'q', 'e' key to move the camera up and down - -import argparse -import os -import platform -import time -from typing import Tuple - -import numpy as np -from matplotlib import pyplot as plt - -import taichi as ti -from taichi.math import uvec3, vec2, vec3 - -try: - import cv2 - import wget - from scipy.spatial.transform import Rotation as R -except: - raise ImportError("The Taichi NGP renderer requires cv2, wget and scipy to be installed.") - - -def depth2img(depth): - depth = (depth - depth.min()) / (depth.max() - depth.min()) - depth_img = cv2.applyColorMap((depth * 255).astype(np.uint8), cv2.COLORMAP_TURBO) - - return depth_img - - -arch = ti.cuda - -if platform.system() == "Darwin": - block_dim = 64 -else: - block_dim = 128 - -sigma_sm_preload = int(128 / block_dim) * 24 -rgb_sm_preload = int(128 / block_dim) * 50 -data_type = ti.f16 -np_type = np.float16 -tf_vec3 = ti.types.vector(3, dtype=data_type) -tf_vec8 = ti.types.vector(8, dtype=data_type) -tf_vec32 = ti.types.vector(32, dtype=data_type) -tf_vec1 = ti.types.vector(1, dtype=data_type) -tf_vec2 = ti.types.vector(2, dtype=data_type) -tf_mat1x3 = ti.types.matrix(1, 3, dtype=data_type) -tf_index_temp = ti.types.vector(8, dtype=ti.i32) - -MAX_SAMPLES = 1024 -NEAR_DISTANCE = 0.01 - -SQRT3 = 1.7320508075688772 -SQRT3_MAX_SAMPLES = SQRT3 / 1024 -SQRT3_2 = 1.7320508075688772 * 2 -PRETRAINED_MODEL_URL = "https://github.com/Linyou/taichi-ngp-renderer/releases/download/v0.1-models/{}.npy" - - -# <----------------- hash table util code -----------------> -@ti.real_func -def calc_dt(t: data_type, exp_step_factor: data_type, grid_size: int, scale: data_type) -> data_type: - return ti.math.clamp(t * exp_step_factor, SQRT3_MAX_SAMPLES, SQRT3_2 * scale / grid_size) - - -@ti.real_func -def __expand_bits(v_: uvec3) -> uvec3: - v = (v_ * ti.uint32(0x00010001)) & ti.uint32(0xFF0000FF) - v = (v * ti.uint32(0x00000101)) & ti.uint32(0x0F00F00F) - v = (v * ti.uint32(0x00000011)) & ti.uint32(0xC30C30C3) - v = (v * ti.uint32(0x00000005)) & ti.uint32(0x49249249) - return v - - -@ti.real_func -def __morton3D(xyz_: uvec3) -> ti.u32: - xyz = __expand_bits(xyz_) - return xyz[0] | (xyz[1] << 1) | (xyz[2] << 2) - - -@ti.real_func -def fast_hash(pos_grid_local: uvec3) -> ti.u32: - result = ti.uint32(0) - primes = uvec3(ti.uint32(1), ti.uint32(2654435761), ti.uint32(805459861)) - for i in ti.static(range(3)): - result ^= ti.uint32(pos_grid_local[i]) * primes[i] - return result - - -@ti.real_func -def under_hash(pos_grid_local: uvec3, resolution: ti.u32) -> ti.u32: - result = ti.uint32(0) - stride = ti.uint32(1) - for i in ti.static(range(3)): - result += ti.uint32(pos_grid_local[i] * stride) - stride *= resolution - return result - - -@ti.real_func -def grid_pos2hash_index(indicator: ti.i32, pos_grid_local: uvec3, resolution: ti.u32, map_size: ti.u32) -> ti.u32: - hash_result = ti.uint32(0) - if indicator == 1: - hash_result = under_hash(pos_grid_local, resolution) - else: - hash_result = fast_hash(pos_grid_local) - - return hash_result % map_size - - -# <----------------- hash table util code -----------------> - - -@ti.real_func -def random_in_unit_disk() -> tf_vec2: - theta = 2.0 * np.pi * ti.random() - return ti.Vector([ti.sin(theta), ti.cos(theta)]) - - -@ti.real_func -def random_normal() -> tf_vec2: - x = ti.random() * 2.0 - 1.0 - y = ti.random() * 2.0 - 1.0 - return tf_vec2(x, y) - - -@ti.real_func -def dir_encode_func(dir_: tf_vec3) -> tf_vec32: - out_feat = tf_vec32(0.0) - dir_n = dir_ / dir_.norm() - x = dir_n[0] - y = dir_n[1] - z = dir_n[2] - xy = x * y - xz = x * z - yz = y * z - x2 = x * x - y2 = y * y - z2 = z * z - - temp = 0.28209479177387814 - out_feat[0] = data_type(temp) - out_feat[1] = data_type(-0.48860251190291987 * y) - out_feat[2] = data_type(0.48860251190291987 * z) - out_feat[3] = data_type(-0.48860251190291987 * x) - out_feat[4] = data_type(1.0925484305920792 * xy) - out_feat[5] = data_type(-1.0925484305920792 * yz) - out_feat[6] = data_type(0.94617469575755997 * z2 - 0.31539156525251999) - out_feat[7] = data_type(-1.0925484305920792 * xz) - out_feat[8] = data_type(0.54627421529603959 * x2 - 0.54627421529603959 * y2) - out_feat[9] = data_type(0.59004358992664352 * y * (-3.0 * x2 + y2)) - out_feat[10] = data_type(2.8906114426405538 * xy * z) - out_feat[11] = data_type(0.45704579946446572 * y * (1.0 - 5.0 * z2)) - out_feat[12] = data_type(0.3731763325901154 * z * (5.0 * z2 - 3.0)) - out_feat[13] = data_type(0.45704579946446572 * x * (1.0 - 5.0 * z2)) - out_feat[14] = data_type(1.4453057213202769 * z * (x2 - y2)) - out_feat[15] = data_type(0.59004358992664352 * x * (-x2 + 3.0 * y2)) - - return out_feat - - -@ti.data_oriented -class NGP_fw: - def __init__(self, scale, cascades, grid_size, base_res, log2_T, res, level, exp_step_factor): - self.res = res - self.N_rays = res[0] * res[1] - self.grid_size = grid_size - self.exp_step_factor = exp_step_factor - self.scale = scale - - # rays intersection parameters - # t1, t2 need to be initialized to -1.0 - self.hits_t = ti.Vector.field(n=2, dtype=data_type, shape=self.N_rays) - self.hits_t.fill(-1.0) - self.center = tf_vec3(0.0, 0.0, 0.0) - self.xyz_min = -tf_vec3(scale, scale, scale) - self.xyz_max = tf_vec3(scale, scale, scale) - self.half_size = (self.xyz_max - self.xyz_min) / 2 - - self.noise_buffer = ti.Vector.field(2, dtype=data_type, shape=self.N_rays) - self.gen_noise_buffer() - - self.rays_o = ti.Vector.field(n=3, dtype=data_type, shape=self.N_rays) - self.rays_d = ti.Vector.field(n=3, dtype=data_type, shape=self.N_rays) - - # use the pre-compute direction and scene pose - self.directions = ti.Matrix.field(n=1, m=3, dtype=data_type, shape=(self.N_rays,)) - self.pose = ti.Matrix.field(n=3, m=4, dtype=data_type, shape=()) - - # density_bitfield is used for point sampling - self.density_bitfield = ti.field(ti.uint32, shape=cascades * grid_size**3 // 32) - - # count the number of rays that still alive - self.counter = ti.field(ti.i32, shape=()) - self.counter[None] = self.N_rays - # current alive buffer index - self.current_index = ti.field(ti.i32, shape=()) - self.current_index[None] = 0 - - # how many samples that need to run the model - self.model_launch = ti.field(ti.i32, shape=()) - - # buffer for the alive rays - self.alive_indices = ti.field(ti.i32, shape=(2 * self.N_rays,)) - - # padd the thread to the factor of block size (thread per block) - self.padd_block_network = ti.field(ti.i32, shape=()) - self.padd_block_composite = ti.field(ti.i32, shape=()) - - # hash table variables - self.min_samples = 1 if exp_step_factor == 0 else 4 - self.per_level_scales = 1.3195079565048218 # hard coded, otherwise it will be have lower percision - self.base_res = base_res - self.max_params = 2**log2_T - self.level = level - # hash table fields - self.offsets = ti.field(ti.i32, shape=(16,)) - self.hash_map_sizes = ti.field(ti.uint32, shape=(16,)) - self.hash_map_indicator = ti.field(ti.i32, shape=(16,)) - - # model parameters - layer1_base = 32 * 64 - layer2_base = layer1_base + 64 * 64 - self.hash_embedding = ti.field(dtype=data_type, shape=(11445040,)) - self.sigma_weights = ti.field(dtype=data_type, shape=(layer1_base + 64 * 16,)) - self.rgb_weights = ti.field(dtype=data_type, shape=(layer2_base + 64 * 8,)) - - # buffers that used for points sampling - self.max_samples_per_rays = 1 - self.max_samples_shape = self.N_rays * self.max_samples_per_rays - - self.xyzs = ti.Vector.field(3, dtype=data_type, shape=(self.max_samples_shape,)) - self.dirs = ti.Vector.field(3, dtype=data_type, shape=(self.max_samples_shape,)) - self.deltas = ti.field(data_type, shape=(self.max_samples_shape,)) - self.ts = ti.field(data_type, shape=(self.max_samples_shape,)) - - # buffers that store the info of sampled points - self.run_model_ind = ti.field(ti.int32, shape=(self.max_samples_shape,)) - self.N_eff_samples = ti.field(ti.int32, shape=(self.N_rays,)) - - # intermediate buffers for network - self.xyzs_embedding = ti.field(data_type, shape=(self.max_samples_shape, 32)) - self.final_embedding = ti.field(data_type, shape=(self.max_samples_shape, 16)) - self.out_3 = ti.field(data_type, shape=(self.max_samples_shape, 3)) - self.out_1 = ti.field(data_type, shape=(self.max_samples_shape,)) - self.temp_hit = ti.field(ti.i32, shape=(self.max_samples_shape,)) - - # results buffers - self.opacity = ti.field(ti.f32, shape=(self.N_rays,)) - self.depth = ti.field(ti.f32, shape=self.N_rays) - self.rgb = ti.Vector.field(3, dtype=ti.f32, shape=(self.N_rays,)) - - # GUI render buffer (data type must be float32) - self.render_buffer = ti.Vector.field( - 3, - dtype=ti.f32, - shape=( - res[0], - res[1], - ), - ) - # camera parameters - self.lookat = np.array([0.0, 0.0, -1.0]) - self.lookat_change = np.zeros((3,)) - self.lookup = np.array([0.0, -1.0, 0.0]) - - def hash_table_init(self): - print( - f"GridEncoding: base resolution: {self.base_res}, log scale per level:{self.per_level_scales:.5f} feature numbers per level: {2} maximum parameters per level: {self.max_params} level: {self.level}" - ) - offset = 0 - for i in range(self.level): - resolution = int(np.ceil(self.base_res * np.exp(i * np.log(self.per_level_scales)) - 1.0)) + 1 - params_in_level = resolution**3 - params_in_level = int(resolution**3) if params_in_level % 8 == 0 else int((params_in_level + 8 - 1) / 8) * 8 - params_in_level = min(self.max_params, params_in_level) - self.offsets[i] = offset - self.hash_map_sizes[i] = params_in_level - self.hash_map_indicator[i] = 1 if resolution**3 <= params_in_level else 0 - offset += params_in_level - - def get_direction(self, camera_angle_x): - w, h = int(self.res[1]), int(self.res[0]) - fx = 0.5 * w / np.tan(0.5 * camera_angle_x) - fy = 0.5 * h / np.tan(0.5 * camera_angle_x) - cx, cy = 0.5 * w, 0.5 * h - - x, y = np.meshgrid( - np.arange(w, dtype=np.float32) + 0.5, - np.arange(h, dtype=np.float32) + 0.5, - indexing="xy", - ) - - directions = np.stack([(x - cx) / fx, (y - cy) / fy, np.ones_like(x)], -1) - - return directions.reshape(-1, 3) - - def load_model(self, model_path): - print(f"Loading model from {model_path}") - model = np.load(model_path, allow_pickle=True).item() - # model = torch.load(model_path, map_location='cpu')['state_dict'] - self.hash_embedding.from_numpy(model["model.xyz_encoder.params"].astype(np_type)) - self.sigma_weights.from_numpy(model["model.xyz_sigmas.params"].astype(np_type)) - self.rgb_weights.from_numpy(model["model.rgb_net.params"].astype(np_type)) - - self.density_bitfield.from_numpy(model["model.density_bitfield"].view("uint32")) - - self.pose.from_numpy(model["poses"][20].astype(np_type)) - if self.res[0] != 800 or self.res[1] != 800: - directions = self.get_direction(model["camera_angle_x"])[:, None, :].astype(np_type) - else: - directions = model["directions"][:, None, :].astype(np_type) - - self.directions.from_numpy(directions) - - @staticmethod - def taichi_init(kernel_profiler): - ti.init( - arch=arch, - offline_cache=True, - kernel_profiler=kernel_profiler, - enable_fallback=False, - ) - - @staticmethod - def taichi_print_profiler(): - ti.profiler.print_kernel_profiler_info() - - @ti.kernel - def reset(self): - self.depth.fill(0.0) - self.opacity.fill(0.0) - self.counter[None] = self.N_rays - for i, j in ti.ndrange(self.N_rays, 2): - self.alive_indices[i * 2 + j] = i - - @ti.real_func - def _ray_aabb_intersec(self, ray_o: tf_vec3, ray_d: tf_vec3) -> tf_vec2: - inv_d = 1.0 / ray_d - - t_min = (self.center - self.half_size - ray_o) * inv_d - t_max = (self.center + self.half_size - ray_o) * inv_d - - _t1 = ti.min(t_min, t_max) - _t2 = ti.max(t_min, t_max) - t1 = _t1.max() - t2 = _t2.min() - - return tf_vec2(t1, t2) - - @ti.kernel - def gen_noise_buffer(self): - for i in range(self.N_rays): - self.noise_buffer[i] = random_normal() - # self.noise_buffer[i] = random_in_unit_disk() - - @ti.kernel - def ray_intersect_dof(self, dist_to_focus: float, len_dis: float): - ti.block_local(self.pose) - for i in self.directions: - c2w = self.pose[None] - dir_ori = self.directions[i] - offset = len_dis * self.noise_buffer[i] - offset_m = tf_mat1x3( - [ - [ - offset[0], - offset[1], - 0.0, - ] - ] - ) - c2w_dir = c2w[:, :3].transpose() - offset_w = offset_m @ c2w_dir - mat_result = (dir_ori * dist_to_focus) @ c2w_dir - offset_w - ray_d = tf_vec3(mat_result[0, 0], mat_result[0, 1], mat_result[0, 2]) - ray_o = c2w[:, 3] + tf_vec3(offset_w[0, 0], offset_w[0, 1], offset_w[0, 2]) - - t1t2 = self._ray_aabb_intersec(ray_o, ray_d) - - if t1t2[1] > 0.0: - self.hits_t[i][0] = data_type(ti.max(t1t2[0], NEAR_DISTANCE)) - self.hits_t[i][1] = t1t2[1] - - self.rays_o[i] = ray_o - self.rays_d[i] = ray_d - - @ti.kernel - def ray_intersect(self): - ti.block_local(self.pose) - for i in self.directions: - c2w = self.pose[None] - mat_result = self.directions[i] @ c2w[:, :3].transpose() - ray_d = tf_vec3(mat_result[0, 0], mat_result[0, 1], mat_result[0, 2]) - ray_o = c2w[:, 3] - - t1t2 = self._ray_aabb_intersec(ray_o, ray_d) - - if t1t2[1] > 0.0: - self.hits_t[i][0] = data_type(ti.max(t1t2[0], NEAR_DISTANCE)) - self.hits_t[i][1] = t1t2[1] - - self.rays_o[i] = ray_o - self.rays_d[i] = ray_d - - @ti.kernel - def raymarching_test_kernel(self, N_samples: int): - self.run_model_ind.fill(0) - for n in ti.ndrange(self.counter[None]): - c_index = self.current_index[None] - r = self.alive_indices[n * 2 + c_index] - grid_size3 = self.grid_size**3 - grid_size_inv = 1.0 / self.grid_size - - ray_o = self.rays_o[r] - ray_d = self.rays_d[r] - t1t2 = self.hits_t[r] - - d_inv = 1.0 / ray_d - - t = t1t2[0] - t2 = t1t2[1] - - s = 0 - - start_idx = n * N_samples - - while (0 <= t) & (t < t2) & (s < N_samples): - # xyz = ray_o + t*ray_d - xyz = ray_o + t * ray_d - dt = calc_dt(t, self.exp_step_factor, self.grid_size, self.scale) - # mip = ti.max(mip_from_pos(xyz, cascades), - # mip_from_dt(dt, grid_size, cascades)) - - mip_bound = 0.5 - mip_bound_inv = 1 / mip_bound - - nxyz = ti.math.clamp( - 0.5 * (xyz * mip_bound_inv + 1) * self.grid_size, - 0.0, - self.grid_size - 1.0, - ) - # nxyz = ti.ceil(nxyz) - - idx = __morton3D(ti.cast(nxyz, ti.u32)) - # occ = density_grid_taichi[idx] > 5.912066756501768 - occ = self.density_bitfield[ti.int32(idx // 32)] & (1 << ti.u32(idx % 32)) - - if occ: - sn = start_idx + s - for p in ti.static(range(3)): - self.xyzs[sn][p] = xyz[p] - self.dirs[sn][p] = ray_d[p] - self.run_model_ind[sn] = 1 - self.ts[sn] = t - self.deltas[sn] = dt - t += dt - self.hits_t[r][0] = t - s += 1 - - else: - txyz = ( - ((nxyz + 0.5 + 0.5 * ti.math.sign(ray_d)) * grid_size_inv * 2 - 1) * mip_bound - xyz - ) * d_inv - - t_target = t + ti.max(0, txyz.min()) - t += calc_dt(t, self.exp_step_factor, self.grid_size, self.scale) - while t < t_target: - t += calc_dt(t, self.exp_step_factor, self.grid_size, self.scale) - - self.N_eff_samples[n] = s - if s == 0: - self.alive_indices[n * 2 + c_index] = -1 - - @ti.kernel - def rearange_index(self, B: ti.i32): - self.model_launch[None] = 0 - - for i in ti.ndrange(B): - if self.run_model_ind[i]: - index = ti.atomic_add(self.model_launch[None], 1) - self.temp_hit[index] = i - - self.model_launch[None] += 1 - self.padd_block_network[None] = ((self.model_launch[None] + block_dim - 1) // block_dim) * block_dim - # self.padd_block_composite[None] = ((self.counter[None]+ 128 - 1)// 128) *128 - - @ti.kernel - def hash_encode(self): - # get hash table embedding - ti.loop_config(block_dim=16) - for sn, level in ti.ndrange(self.model_launch[None], 16): - # normalize to [0, 1], before is [-0.5, 0.5] - xyz = self.xyzs[self.temp_hit[sn]] + 0.5 - offset = self.offsets[level] * 2 - indicator = self.hash_map_indicator[level] - map_size = self.hash_map_sizes[level] - - init_val0 = tf_vec1(0.0) - init_val1 = tf_vec1(1.0) - local_feature_0 = init_val0[0] - local_feature_1 = init_val0[0] - - index_temp = tf_index_temp(0) - w_temp = tf_vec8(0.0) - hash_temp_1 = tf_vec8(0.0) - hash_temp_2 = tf_vec8(0.0) - - scale = self.base_res * ti.exp(level * ti.log(self.per_level_scales)) - 1.0 - resolution = ti.cast(ti.ceil(scale), ti.uint32) + 1 - - pos = xyz * scale + 0.5 - pos_grid_uint = ti.cast(ti.floor(pos), ti.uint32) - pos -= pos_grid_uint - # pos_grid_uint = ti.cast(pos_grid, ti.uint32) - - for idx in ti.static(range(8)): - # idx_uint = ti.cast(idx, ti.uint32) - w = init_val1[0] - pos_grid_local = uvec3(0) - - for d in ti.static(range(3)): - if (idx & (1 << d)) == 0: - pos_grid_local[d] = pos_grid_uint[d] - w *= data_type(1 - pos[d]) - else: - pos_grid_local[d] = pos_grid_uint[d] + 1 - w *= data_type(pos[d]) - - index = ti.int32(grid_pos2hash_index(indicator, pos_grid_local, resolution, map_size)) - index_temp[idx] = offset + index * 2 - w_temp[idx] = w - - for idx in ti.static(range(8)): - hash_temp_1[idx] = self.hash_embedding[index_temp[idx]] - hash_temp_2[idx] = self.hash_embedding[index_temp[idx] + 1] - - for idx in ti.static(range(8)): - local_feature_0 += data_type(w_temp[idx] * hash_temp_1[idx]) - local_feature_1 += data_type(w_temp[idx] * hash_temp_2[idx]) - - self.xyzs_embedding[sn, level * 2] = local_feature_0 - self.xyzs_embedding[sn, level * 2 + 1] = local_feature_1 - - @ti.kernel - def sigma_layer(self): - ti.loop_config(block_dim=block_dim) - for sn in ti.ndrange(self.padd_block_network[None]): - tid = sn % block_dim - did_launch_num = self.model_launch[None] - init_val = tf_vec1(0.0) - xyz_feat = ti.simt.block.SharedArray((32, block_dim), data_type) - weight = ti.simt.block.SharedArray((64 * 32 + 64 * 16,), data_type) - hid1 = ti.simt.block.SharedArray((64, block_dim), data_type) - hid2 = ti.simt.block.SharedArray((16, block_dim), data_type) - for i in ti.static(range(sigma_sm_preload)): - k = tid * sigma_sm_preload + i - weight[k] = self.sigma_weights[k] - ti.simt.block.sync() - - if sn < did_launch_num: - for i in ti.static(range(32)): - xyz_feat[i, tid] = self.xyzs_embedding[sn, i] - - for i in range(64): - temp = init_val[0] - for j in ti.static(range(32)): - temp += xyz_feat[j, tid] * weight[i * 32 + j] - - hid1[i, tid] = temp - ti.simt.block.sync() - - for i in range(16): - temp = init_val[0] - for j in ti.static(range(64)): - temp += data_type(ti.max(0.0, hid1[j, tid])) * weight[64 * 32 + i * 64 + j] - hid2[i, tid] = temp - ti.simt.block.sync() - - self.out_1[self.temp_hit[sn]] = data_type(ti.exp(hid2[0, tid])) - for i in ti.static(range(16)): - self.final_embedding[sn, i] = hid2[i, tid] - - ti.simt.block.sync() - - @ti.kernel - def rgb_layer(self): - ti.loop_config(block_dim=block_dim) - for sn in ti.ndrange(self.padd_block_network[None]): - ray_id = self.temp_hit[sn] - tid = sn % block_dim - did_launch_num = self.model_launch[None] - init_val = tf_vec1(0.0) - weight = ti.simt.block.SharedArray((64 * 32 + 64 * 64 + 64 * 4,), data_type) - hid1 = ti.simt.block.SharedArray((64, block_dim), data_type) - hid2 = ti.simt.block.SharedArray((64, block_dim), data_type) - for i in ti.static(range(rgb_sm_preload)): - k = tid * rgb_sm_preload + i - weight[k] = self.rgb_weights[k] - ti.simt.block.sync() - - if sn < did_launch_num: - dir_ = self.dirs[ray_id] - dir_feat = dir_encode_func(dir_) - - for i in ti.static(range(16)): - dir_feat[16 + i] = self.final_embedding[sn, i] - - for i in range(64): - temp = init_val[0] - for j in ti.static(range(32)): - temp += dir_feat[j] * weight[i * 32 + j] - - hid1[i, tid] = temp - ti.simt.block.sync() - - for i in range(64): - temp = init_val[0] - for j in ti.static(range(64)): - temp += data_type(ti.max(0.0, hid1[j, tid])) * weight[64 * 32 + i * 64 + j] - - hid2[i, tid] = temp - ti.simt.block.sync() - - for i in ti.static(range(3)): - temp = init_val[0] - for j in ti.static(range(64)): - temp += data_type(ti.max(0.0, hid2[j, tid])) * weight[64 * 32 + 64 * 64 + i * 64 + j] - - hid1[i, tid] = temp - ti.simt.block.sync() - - for i in ti.static(range(3)): - self.out_3[self.temp_hit[sn], i] = data_type(1 / (1 + ti.exp(-hid1[i, tid]))) - ti.simt.block.sync() - - @ti.kernel - def FullyFusedMLP(self): - ti.loop_config(block_dim=block_dim) - for sn in ti.ndrange(self.padd_block_network[None]): - ray_id = self.temp_hit[sn] - tid = sn % block_dim - did_launch_num = self.model_launch[None] - init_val = tf_vec1(0.0) - xyz_feat = tf_vec32(0.0) - weight = ti.simt.block.SharedArray((64 * 32 + 64 * 64 + 64 * 4,), data_type) - hid2_2 = ti.simt.block.SharedArray((32 * block_dim,), data_type) - hid2_1 = ti.simt.block.SharedArray((32 * block_dim,), data_type) - hid1 = ti.simt.block.SharedArray((64 * block_dim,), data_type) - for i in ti.static(range(rgb_sm_preload)): - k = tid * rgb_sm_preload + i - weight[k] = self.rgb_weights[k] - for i in ti.static(range(sigma_sm_preload)): - k = tid * sigma_sm_preload + i - hid2_1[k] = self.sigma_weights[k] - ti.simt.block.sync() - - if sn < did_launch_num: - dir_ = self.dirs[ray_id] - for i in ti.static(range(32)): - xyz_feat[i] = self.xyzs_embedding[sn, i] - dir_feat = dir_encode_func(dir_) - - for i in range(64): - temp = init_val[0] - for j in ti.static(range(32)): - temp += xyz_feat[j] * hid2_1[i * 32 + j] - hid1[i * block_dim + tid] = temp - ti.simt.block.sync() - - for i in range(16): - temp = init_val[0] - for j in ti.static(range(64)): - temp += data_type(ti.max(0.0, hid1[j * block_dim + tid])) * hid2_1[64 * 32 + i * 64 + j] - hid2_2[i * block_dim + tid] = temp - ti.simt.block.sync() - - out1 = data_type(ti.exp(hid2_2[tid])) - - for i in ti.static(range(16)): - dir_feat[16 + i] = hid2_2[i * block_dim + tid] - - for i in range(64): - temp = init_val[0] - for j in ti.static(range(32)): - temp += dir_feat[j] * weight[i * 32 + j] - hid1[i * block_dim + tid] = temp - ti.simt.block.sync() - - for i in range(32): - temp1 = init_val[0] - temp2 = init_val[0] - for j in ti.static(range(64)): - temp1 += data_type(ti.max(0.0, hid1[j * block_dim + tid])) * weight[64 * 32 + i * 64 + j] - temp2 += data_type(ti.max(0.0, hid1[j * block_dim + tid])) * weight[64 * 32 + (i + 32) * 64 + j] - hid2_1[i * block_dim + tid] = temp1 - hid2_2[i * block_dim + tid] = temp2 - ti.simt.block.sync() - - for i in ti.static(range(3)): - temp = init_val[0] - for j in ti.static(range(32)): - temp += ( - data_type(ti.max(0.0, hid2_1[j * block_dim + tid])) * weight[64 * 32 + 64 * 64 + i * 64 + j] - ) - # ti.simt.block.sync() - temp += ( - data_type(ti.max(0.0, hid2_2[j * block_dim + tid])) - * weight[64 * 32 + 64 * 64 + i * 64 + j + 32] - ) - hid1[i * block_dim + tid] = temp - ti.simt.block.sync() - - self.out_1[self.temp_hit[sn]] = out1 - for i in ti.static(range(3)): - self.out_3[self.temp_hit[sn], i] = data_type(1 / (1 + ti.exp(-hid1[i * block_dim + tid]))) - ti.simt.block.sync() - - @ti.kernel - def composite_test(self, max_samples: ti.i32, T_threshold: data_type): - for n in ti.ndrange(self.counter[None]): - N_samples = self.N_eff_samples[n] - if N_samples != 0: - c_index = self.current_index[None] - r = self.alive_indices[n * 2 + c_index] - - T = data_type(1.0 - self.opacity[r]) - - start_idx = n * max_samples - - rgb_temp = tf_vec3(0.0) - depth_temp = tf_vec1(0.0) - opacity_temp = tf_vec1(0.0) - out_3_temp = tf_vec3(0.0) - - for s in range(N_samples): - sn = start_idx + s - a = data_type(1.0 - ti.exp(-self.out_1[sn] * self.deltas[sn])) - w = a * T - - for i in ti.static(range(3)): - out_3_temp[i] = self.out_3[sn, i] - - rgb_temp += w * out_3_temp - depth_temp[0] += w * self.ts[sn] - opacity_temp[0] += w - - T *= data_type(1.0 - a) - - if T <= T_threshold: - self.alive_indices[n * 2 + c_index] = -1 - break - - self.rgb[r] += rgb_temp - self.depth[r] += depth_temp[0] - self.opacity[r] += opacity_temp[0] - - @ti.kernel - def re_order(self, B: ti.i32): - self.counter[None] = 0 - c_index = self.current_index[None] - n_index = (c_index + 1) % 2 - self.current_index[None] = n_index - - for i in ti.ndrange(B): - alive_temp = self.alive_indices[i * 2 + c_index] - if alive_temp >= 0: - index = ti.atomic_add(self.counter[None], 1) - self.alive_indices[index * 2 + n_index] = alive_temp - - def write_image(self): - rgb_np = self.rgb.to_numpy().reshape(self.res[0], self.res[1], 3) - depth_np = self.depth.to_numpy().reshape(self.res[0], self.res[1]) - plt.imsave("taichi_ngp.png", (rgb_np * 255).astype(np.uint8)) - plt.imsave("taichi_ngp_depth.png", depth2img(depth_np)) - - def render(self, max_samples, T_threshold, use_dof=False, dist_to_focus=0.8, len_dis=0.0) -> Tuple[float, int, int]: - samples = 0 - self.reset() - self.gen_noise_buffer() - if use_dof: - self.ray_intersect_dof(dist_to_focus, len_dis) - else: - self.ray_intersect() - - while samples < max_samples: - N_alive = self.counter[None] - if N_alive == 0: - break - - # how many more samples the number of samples add for each ray - N_samples = max(min(self.N_rays // N_alive, 64), self.min_samples) - samples += N_samples - launch_model_total = N_alive * N_samples - - self.raymarching_test_kernel(N_samples) - self.rearange_index(launch_model_total) - # self.dir_encode() - self.hash_encode() - self.sigma_layer() - self.rgb_layer() - # self.FullyFusedMLP() - self.composite_test(N_samples, T_threshold) - self.re_order(N_alive) - - return samples, N_alive, N_samples - - def render_frame(self, frame_id): - t = time.time() - samples, N_alive, N_samples = self.render(max_samples=100, T_threshold=1e-4) - self.write_image() - - print(f"samples: {samples}, N_alive: {N_alive}, N_samples: {N_samples}") - print(f"Render time: {1000*(time.time()-t):.2f} ms") - - @ti.kernel - def rgb_to_render_buffer(self, frame: ti.i32): - for i, j in self.render_buffer: - rgb = self.rgb[(self.res[0] - j) * self.res[1] + i] - self.render_buffer[i, j] = rgb / frame - - @ti.kernel - def depth_max(self) -> vec2: - max_v = self.depth[0] - min_v = self.depth[0] - for i in ti.ndrange(self.N_rays): - ti.atomic_max(max_v, self.depth[i]) - ti.atomic_min(min_v, self.depth[i]) - return vec2(max_v, min_v) - - @ti.kernel - def depth_to_render_buffer(self, max_min: vec2): - for i, j in self.render_buffer: - max_v = max_min[0] - min_v = max_min[1] - depth = self.depth[(self.res[0] - j) * self.res[1] + i] - pixel = (vec3(depth) - min_v) / (max_v - min_v) - self.render_buffer[i, j] = pixel - - def init_cam(self): - self.lookat = self.lookat @ self.pose.to_numpy()[:, :3].T - - def render_gui(self): - video_manager = None - - # check if the export file exists for snapshot and video - export_dir = "./export/" - if not os.path.exists(export_dir): - os.mkdir(export_dir) - - W, H = self.res - window = ti.ui.Window("Taichi NGP", (W, H)) - canvas = window.get_canvas() - gui = window.get_gui() - - last_mouse_x = None - last_mouse_y = None - rotate_speed = 50 - movement_speed = 0.03 - max_samples_for_rendering = 100 - render_time = 0 - # white_bg = False - recording = False - show_depth = False - use_dof = False - last_use_dof = False - frame = 0 - T_threshold = 1e-2 - dist_to_focus = 1.2 - len_dis = 0.04 - self.init_cam() - last_pose = self.pose.to_numpy() - total_frame = 0 - last_dist_to_focus = dist_to_focus - last_len_dis = len_dis - - while window.running: - # TODO: make it more efficient - pose = self.pose.to_numpy() - total_frame += 1 - if not window.is_pressed(ti.ui.RMB): - last_mouse_x = None - last_mouse_y = None - else: - curr_mouse_x, curr_mouse_y = window.get_cursor_pos() - if last_mouse_x is None or last_mouse_y is None: - last_mouse_x, last_mouse_y = curr_mouse_x, curr_mouse_y - else: - dx = curr_mouse_x - last_mouse_x - dy = curr_mouse_y - last_mouse_y - rotvec_x = pose[:, 1] * np.radians(rotate_speed * dx) - rotvec_y = pose[:, 0] * np.radians(rotate_speed * dy) - pose = R.from_rotvec(rotvec_x).as_matrix() @ R.from_rotvec(rotvec_y).as_matrix() @ pose - last_mouse_x, last_mouse_y = curr_mouse_x, curr_mouse_y - correct_dir = 1.0 if pose[2, 3] < 0.0 else -1.0 - self.lookat = np.array([0.0, 0.0, correct_dir]) @ pose[:, :3].T - - front = self.lookat - pose[:, 3] - front = front / np.linalg.norm(front) - up = self.lookup @ pose[:, :3].T - left = np.cross(up, front) - position_change = np.zeros(3) - if window.is_pressed("w"): - position_change = front * movement_speed - if window.is_pressed("s"): - position_change = -front * movement_speed - if window.is_pressed("a"): - position_change = left * movement_speed - if window.is_pressed("d"): - position_change = -left * movement_speed - if window.is_pressed("e"): - position_change = up * movement_speed - if window.is_pressed("q"): - position_change = -up * movement_speed - pose[:, 3] += position_change - self.lookat += position_change - if (last_pose - pose).sum(): - last_pose = pose - self.pose.from_numpy(pose.astype(np.float16)) - self.rgb.fill(0.0) - total_frame = 1 - - with gui.sub_window("Options", 0.05, 0.05, 0.68, 0.3) as w: - w.text("General") - T_threshold = w.slider_float("transparency threshold", T_threshold, 0.0, 1.0) - max_samples_for_rendering = w.slider_float("max samples", max_samples_for_rendering, 1, 100) - show_depth = w.checkbox("show depth", show_depth) - # white_bg = w.checkbox("white background", white_bg) - - w.text("Camera") - use_dof = w.checkbox("apply depth of field", use_dof) - dist_to_focus = w.slider_float("focus distance", dist_to_focus, 0.8, 3.0) - len_dis = w.slider_float("lens size", len_dis, 0.0, 0.1) - if last_dist_to_focus != dist_to_focus or last_len_dis != len_dis or last_use_dof != use_dof: - last_dist_to_focus = dist_to_focus - last_len_dis = len_dis - last_use_dof = use_dof - self.rgb.fill(0.0) - total_frame = 1 - - w.text(f"Render time: {render_time:.2f} ms") - - with gui.sub_window("Export", 0.75, 0.05, 0.2, 0.1) as w: - if gui.button("snapshot "): - ti.tools.imwrite(self.render_buffer.to_numpy(), export_dir + "snap_shot.png") - print("save snapshot in export folder") - if gui.button("recording"): - frame = 0 - if not recording: - video_manager = ti.tools.VideoManager( - output_dir=export_dir, framerate=24, automatic_build=False - ) - recording = True - else: - recording = False - video_manager.make_video(gif=True, mp4=True) - print("save video in export folder") - - if recording and video_manager: - w.text(f"recording frames: {frame}") - frame += 1 - pixels_img = self.render_buffer.to_numpy() - video_manager.write_frame(pixels_img) - - t = time.time() - _, _, _ = self.render( - max_samples=max_samples_for_rendering, - T_threshold=T_threshold, - use_dof=use_dof, - dist_to_focus=dist_to_focus, - len_dis=len_dis, - ) - - if not show_depth: - self.rgb_to_render_buffer(total_frame) - else: - self.depth_to_render_buffer(self.depth_max()) - - render_time = 1000 * (time.time() - t) - canvas.set_image(self.render_buffer) - window.show() - - -def main(args): - NGP_fw.taichi_init(args.print_profile) - res = args.res - scale = 0.5 - ngp = NGP_fw( - scale=scale, - cascades=max(1 + int(np.ceil(np.log2(2 * scale))), 1), - grid_size=128, - base_res=16, - log2_T=19, - res=[res, res], - level=16, - exp_step_factor=0, - ) - if args.model_path: - ngp.load_model(args.model_path) - else: - model_dir = "./npy_models/" - if not os.path.exists(model_dir): - os.mkdir(model_dir) - npy_file = os.path.join(model_dir, args.scene + ".npy") - if not os.path.exists(npy_file): - print(f"No {args.scene} model found, downloading ...") - url = PRETRAINED_MODEL_URL.format(args.scene) - wget.download(url, out=npy_file) - ngp.load_model(npy_file) - - ngp.hash_table_init() - - if not args.gui: - ngp.render_frame(0) - else: - ngp.render_gui() - - if args.print_profile: - NGP_fw.taichi_print_profiler() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--res", type=int, default=800) - parser.add_argument( - "--scene", - type=str, - default="lego", - choices=[ - "ship", - "mic", - "materials", - "lego", - "hotdog", - "ficus", - "drums", - "chair", - ], - ) - parser.add_argument("--model_path", type=str, default=None) - parser.add_argument("--gui", action="store_true", default=False) - parser.add_argument("--print_profile", action="store_true", default=False) - cmd_args, _ = parser.parse_known_args() - main(cmd_args) diff --git a/requirements_test.txt b/requirements_test.txt index 0f9cea2676..5c8d3e4fde 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -16,4 +16,3 @@ nbmake marko PyYAML pyevtk -wget From ba64784e3ecdb97c40a6c517aed110b5225483ee Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 12:29:39 -0700 Subject: [PATCH 46/68] add pyi to wheel --- pyproject.toml | 6 +++--- setup.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7c577c396b..5ad5014d48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [build-system] requires = [ - "setuptools", + "setuptools>=68.0.0", "wheel", "numpy", "pybind11", - "cmake", + "pybind11-stubgen", + "cmake>=3.11", "scikit-build", "ninja; platform_system != 'Windows'", - "setuptools>=68.0.0", "cffi>=1.16.0", ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 734d49876b..c1a4d7f9dd 100644 --- a/setup.py +++ b/setup.py @@ -8,12 +8,14 @@ import glob import multiprocessing import os +from os import path import platform import shutil import subprocess import sys from distutils.command.clean import clean from distutils.dir_util import remove_tree +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools import find_packages from skbuild import setup @@ -109,6 +111,34 @@ def run(self): os.remove(f) +class BDistWheelWithStubs(_bdist_wheel): + def run(self): + build_lib = self.get_finalized_command('build_py').build_lib + taichi_path = path.join(path.dirname(path.dirname(build_lib)), "cmake-install/python") + env = os.environ.copy() + env['PYTHONPATH'] = taichi_path + os.pathsep + env.get('PYTHONPATH', '') + + # command that works: + # PYTHONPATH=_skbuild/linux-x86_64-3.10/cmake-install/python pybind11-stubgen \ + # taichi._lib.core.taichi_python --ignore-all-errors + cmd_line = [ + "pybind11-stubgen", + "taichi._lib.core.taichi_python", + "--ignore-all-errors" + ] + print(" ".join(cmd_line)) + subprocess.check_call(cmd_line, env=env) + stub_filepath = "stubs/taichi/_lib/core/taichi_python.pyi" + target_filepath = path.join(build_lib, "taichi/_lib/core/taichi_python.pyi") + py_typed_dst = os.path.join(build_lib, "taichi/_lib/core/py.typed") + os.makedirs(os.path.dirname(target_filepath), exist_ok=True) + print("copying ", stub_filepath, "to", target_filepath) + shutil.copy(stub_filepath, target_filepath) + with open(py_typed_dst, "w"): + pass # creates an empty file + super().run() + + def get_cmake_args(): import shlex @@ -234,6 +264,9 @@ def sign_development_for_apple_m1(): data_files=[ (os.path.join("_lib", "runtime"), data_files), ], + package_data={ + "taichi._lib.core": ["taichi_python.pyi", "py.typed"], + }, keywords=["graphics", "simulation"], license="Apache Software License (http://www.apache.org/licenses/LICENSE-2.0)", include_package_data=True, @@ -245,7 +278,11 @@ def sign_development_for_apple_m1(): classifiers=classifiers, cmake_args=get_cmake_args(), cmake_process_manifest_hook=cmake_install_manifest_filter, - cmdclass={"egg_info": EggInfo, "clean": Clean}, + cmdclass={ + "egg_info": EggInfo, + "clean": Clean, + "bdist_wheel": BDistWheelWithStubs, + }, has_ext_modules=lambda: True, ) From 42a61d4df58e2885f8818d79f85d59d430321ae3 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 12:43:00 -0700 Subject: [PATCH 47/68] add test of typing --- requirements_test.txt | 1 + tests/python/test_pyi_stubs.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/python/test_pyi_stubs.py diff --git a/requirements_test.txt b/requirements_test.txt index 5c8d3e4fde..b3d8165443 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -16,3 +16,4 @@ nbmake marko PyYAML pyevtk +mypy diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py new file mode 100644 index 0000000000..dd83d7a512 --- /dev/null +++ b/tests/python/test_pyi_stubs.py @@ -0,0 +1,24 @@ +from tests import test_utils +import tempfile +import sys +import os +import subprocess + + +@test_utils.test() +def test_pyi_stubs(): + test_code = """ +import taichi._lib.core.taichi_python +reveal_type(taichi._lib.core.taichi_python) +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = os.path.join(tmpdir, "tmp_mypy_test.py") + with open(test_file, "w") as f: + f.write(test_code) + + # if no types, this will fail + subprocess.check_output([ + sys.executable, + "-m", "mypy", + test_file, + ]) From cc94e1738f16798fae3deae6562655c67b1c6b4d Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 12:44:11 -0700 Subject: [PATCH 48/68] pre-commit --- setup.py | 10 +++------- tests/python/test_pyi_stubs.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index c1a4d7f9dd..e5f26d58cc 100644 --- a/setup.py +++ b/setup.py @@ -113,19 +113,15 @@ def run(self): class BDistWheelWithStubs(_bdist_wheel): def run(self): - build_lib = self.get_finalized_command('build_py').build_lib + build_lib = self.get_finalized_command("build_py").build_lib taichi_path = path.join(path.dirname(path.dirname(build_lib)), "cmake-install/python") env = os.environ.copy() - env['PYTHONPATH'] = taichi_path + os.pathsep + env.get('PYTHONPATH', '') + env["PYTHONPATH"] = taichi_path + os.pathsep + env.get("PYTHONPATH", "") # command that works: # PYTHONPATH=_skbuild/linux-x86_64-3.10/cmake-install/python pybind11-stubgen \ # taichi._lib.core.taichi_python --ignore-all-errors - cmd_line = [ - "pybind11-stubgen", - "taichi._lib.core.taichi_python", - "--ignore-all-errors" - ] + cmd_line = ["pybind11-stubgen", "taichi._lib.core.taichi_python", "--ignore-all-errors"] print(" ".join(cmd_line)) subprocess.check_call(cmd_line, env=env) stub_filepath = "stubs/taichi/_lib/core/taichi_python.pyi" diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py index dd83d7a512..0b4dbf3fa0 100644 --- a/tests/python/test_pyi_stubs.py +++ b/tests/python/test_pyi_stubs.py @@ -17,8 +17,11 @@ def test_pyi_stubs(): f.write(test_code) # if no types, this will fail - subprocess.check_output([ - sys.executable, - "-m", "mypy", - test_file, - ]) + subprocess.check_output( + [ + sys.executable, + "-m", + "mypy", + test_file, + ] + ) From 9fecadd60bb563667a45162217ddd1a39f985fa9 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 12:44:46 -0700 Subject: [PATCH 49/68] ruff --- setup.py | 4 ++-- tests/python/test_pyi_stubs.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index e5f26d58cc..06af39f71d 100644 --- a/setup.py +++ b/setup.py @@ -8,18 +8,18 @@ import glob import multiprocessing import os -from os import path import platform import shutil import subprocess import sys from distutils.command.clean import clean from distutils.dir_util import remove_tree -from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +from os import path from setuptools import find_packages from skbuild import setup from skbuild.command.egg_info import egg_info +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel root_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py index 0b4dbf3fa0..28dd19dfc0 100644 --- a/tests/python/test_pyi_stubs.py +++ b/tests/python/test_pyi_stubs.py @@ -1,8 +1,9 @@ -from tests import test_utils -import tempfile -import sys import os import subprocess +import sys +import tempfile + +from tests import test_utils @test_utils.test() From e58378a63362bd19c923ce521f6597b090139bb6 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 14:36:58 -0700 Subject: [PATCH 50/68] api doc --- .github/workflows/api_doc.yml | 36 +++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index c54921c5dd..f6b393720c 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -1,9 +1,13 @@ name: api docs build on: - pull_request: - branches: - - main + # pull_request: + # branches: + # - main + workflow_run: + workflows: ["Manylinux wheel build"] + types: + - completed workflow_dispatch: concurrency: @@ -14,14 +18,34 @@ permissions: jobs: deploy: runs-on: ubuntu-latest + environment: aws_storage steps: - uses: actions/checkout@v4 - - - name: Build + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Download artifact + run: | + set -x + aws s3 sync s3://genesis-ai-compiler/ci/taichi/${GITHUB_SHA:0:7}/cp310/ dist/ + - name: Python check + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: manylinux install wheel + run: | + set -x + ls -lh dist/ + pip install dist/*.whl + python -c 'import taichi as ti; ti.init(arch=ti.cpu); print(ti.__version__)' + - name: Build docs run: | pip install pdoc pdoc --output-dir public taichi - - name: Deploy + - name: Deploy docs uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} From b3276bede8c304baa5a9d177acd42eb539ce361e Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 14:55:18 -0700 Subject: [PATCH 51/68] pre-commit --- .github/workflows/api_doc.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index f6b393720c..f33f52febc 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -1,17 +1,14 @@ name: api docs build on: - # pull_request: - # branches: - # - main workflow_run: workflows: ["Manylinux wheel build"] types: - - completed + - completed workflow_dispatch: concurrency: - group: gh-pages-deploy-${{ github.ref }} + group: gh-pages-deploy-${{ github.head_ref }} cancel-in-progress: true permissions: contents: write From 374a6582ac2637a16c0d6c493d2822841469b437 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 15:30:23 -0700 Subject: [PATCH 52/68] enable vulkan --- .github/workflows/scripts_new/manylinux_wheel/2_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index 3e2933fc0b..ecc009a04e 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -4,4 +4,5 @@ set -ex export PATH=$PWD/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4/bin:$PATH +export TAICHI_CMAKE_ARGS="-DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_CUDA:BOOL=ON" ./build.py wheel From f4b825010769f9f58402c48a80b32ea4be23ca27 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 15:33:59 -0700 Subject: [PATCH 53/68] -v -r 3 --- .github/workflows/scripts_new/manylinux_wheel/5_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh index 27065227ca..c993faf7d7 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/5_test.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/5_test.sh @@ -3,4 +3,4 @@ set -ex pip install -r requirements_test.txt -python tests/run_tests.py +python tests/run_tests.py -v -r 3 From f0054ef0d4ad41c668309938c91c661ff767d086 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 16:46:55 -0700 Subject: [PATCH 54/68] Revert "enable vulkan" This reverts commit 374a6582ac2637a16c0d6c493d2822841469b437. --- .github/workflows/scripts_new/manylinux_wheel/2_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index ecc009a04e..3e2933fc0b 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -4,5 +4,4 @@ set -ex export PATH=$PWD/clang+llvm-15.0.4-x86_64-linux-gnu-rhel-8.4/bin:$PATH -export TAICHI_CMAKE_ARGS="-DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_CUDA:BOOL=ON" ./build.py wheel From c2e4ceb914b8883b44716734f96bff7332a49679 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Sun, 29 Jun 2025 18:13:29 -0700 Subject: [PATCH 55/68] add __init__.py so can continue to build with develop --- python/taichi/_lib/core/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/taichi/_lib/core/__init__.py diff --git a/python/taichi/_lib/core/__init__.py b/python/taichi/_lib/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 40aa8bd2c832e59454a59cb51d2dc34db2842f74 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 14:16:17 -0700 Subject: [PATCH 56/68] use workflow_run --- .github/workflows/api_doc.yml | 11 ++++++----- .github/workflows/manylinux_wheel.yml | 11 +++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index f33f52febc..e255cdb92d 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -1,12 +1,13 @@ name: api docs build on: - workflow_run: - workflows: ["Manylinux wheel build"] - types: - - completed + workflow_call: + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true workflow_dispatch: - concurrency: group: gh-pages-deploy-${{ github.head_ref }} cancel-in-progress: true diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 512a2c5056..e078860a6b 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -33,7 +33,7 @@ jobs: name: manylinux_wheel_${{ matrix.PYTHON_CP_VERSION }} path: wheelhouse/*.whl - upload_to_storaage: + upload_to_storage: name: Manylinux wheel upload runs-on: ubuntu-22.04 needs: build_wheel @@ -63,7 +63,7 @@ jobs: test_wheel: name: Manylinux wheel Test runs-on: ubuntu-22.04 - needs: upload_to_storaage + needs: upload_to_storage environment: aws_storage strategy: matrix: @@ -101,3 +101,10 @@ jobs: - name: manylinux test run: | bash .github/workflows/scripts_new/manylinux_wheel/5_test.sh + + api_docs: + uses: ./.github/workflows/api_doc.yml + needs: upload_to_storage + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From e5cf511966146dcd6b911d9595ef007d68721c43 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 15:30:58 -0700 Subject: [PATCH 57/68] dummy From 4144b402282d594a30b14cf8db0983adccd91be8 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 15:36:41 -0700 Subject: [PATCH 58/68] comment out api_docs --- .github/workflows/manylinux_wheel.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index e078860a6b..449700175a 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -102,9 +102,9 @@ jobs: run: | bash .github/workflows/scripts_new/manylinux_wheel/5_test.sh - api_docs: - uses: ./.github/workflows/api_doc.yml - needs: upload_to_storage - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # api_docs: + # uses: ./.github/workflows/api_doc.yml + # needs: upload_to_storage + # secrets: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 667a08f55f3d12a7ca3e07a2c12edebbf174e285 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 15:56:11 -0700 Subject: [PATCH 59/68] re add api_docs --- .github/workflows/manylinux_wheel.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index 449700175a..d4d4d72d62 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -101,10 +101,9 @@ jobs: - name: manylinux test run: | bash .github/workflows/scripts_new/manylinux_wheel/5_test.sh - - # api_docs: - # uses: ./.github/workflows/api_doc.yml - # needs: upload_to_storage - # secrets: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + api_docs: + uses: ./.github/workflows/api_doc.yml + needs: upload_to_storage + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 55526dbc0f79a503eefe0c31da594ebef14e5d47 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 16:07:44 -0700 Subject: [PATCH 60/68] pass in head ref --- .github/workflows/api_doc.yml | 9 +++++++-- .github/workflows/manylinux_wheel.yml | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index e255cdb92d..cf2a6ac8a7 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -2,6 +2,11 @@ name: api docs build on: workflow_call: + inputs: + head_ref: + description: 'The head ref (branch name or PR ref)' + required: true + type: string secrets: AWS_ACCESS_KEY_ID: required: true @@ -9,7 +14,7 @@ on: required: true workflow_dispatch: concurrency: - group: gh-pages-deploy-${{ github.head_ref }} + group: gh-pages-deploy-${{ github.event.inputs.head_ref || github.head_ref }} cancel-in-progress: true permissions: contents: write @@ -48,6 +53,6 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./public - destination_dir: ${{ github.head_ref }} + destination_dir: ${{ github.event.inpus.head_ref || github.head_ref }} keep_files: true publish_branch: gh-pages diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index d4d4d72d62..dfb235d48c 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -104,6 +104,8 @@ jobs: api_docs: uses: ./.github/workflows/api_doc.yml needs: upload_to_storage + with: + head_ref: ${{ github.head_ref }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 9e6961e709a7326741899a97664e09de43fbf9c8 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 16:11:42 -0700 Subject: [PATCH 61/68] inputs --- .github/workflows/api_doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index cf2a6ac8a7..1912d83110 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -53,6 +53,6 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./public - destination_dir: ${{ github.event.inpus.head_ref || github.head_ref }} + destination_dir: ${{ github.event.inputs.head_ref || github.head_ref }} keep_files: true publish_branch: gh-pages From 00195c36faf97b26b83bf3fd1e1d37e503efe7aa Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 30 Jun 2025 16:14:43 -0700 Subject: [PATCH 62/68] cnotents write --- .github/workflows/manylinux_wheel.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index dfb235d48c..d1f2e92292 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -4,6 +4,8 @@ on: branches: - main workflow_dispatch: +permissions: + contents: write jobs: build_wheel: name: Manylinux wheel Build From 434609e33df4f507be614d91f0c6d42cf02275ed Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 1 Jul 2025 10:46:49 -0700 Subject: [PATCH 63/68] fix tset_pyi_stubs to use pyright intead of mypy --- requirements_test.txt | 1 + tests/python/test_pyi_stubs.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements_test.txt b/requirements_test.txt index 5c8d3e4fde..0a3c18a99b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -16,3 +16,4 @@ nbmake marko PyYAML pyevtk +pyright diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py index 28dd19dfc0..7fc2b74f91 100644 --- a/tests/python/test_pyi_stubs.py +++ b/tests/python/test_pyi_stubs.py @@ -17,12 +17,12 @@ def test_pyi_stubs(): with open(test_file, "w") as f: f.write(test_code) - # if no types, this will fail - subprocess.check_output( + res = subprocess.check_output( [ sys.executable, "-m", - "mypy", + "pyright", test_file, ] - ) + ).decode("utf-8") + assert "unknown" not in res.lower() From 30a7d054f906a390cdfd268596931be47f605ac1 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Jul 2025 07:22:15 -0700 Subject: [PATCH 64/68] address review comments --- setup.py | 9 ++++----- tests/python/test_pyi_stubs.py | 21 +++++---------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index 06af39f71d..77b66eee25 100644 --- a/setup.py +++ b/setup.py @@ -14,12 +14,11 @@ import sys from distutils.command.clean import clean from distutils.dir_util import remove_tree -from os import path from setuptools import find_packages from skbuild import setup from skbuild.command.egg_info import egg_info -from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +from wheel.bdist_wheel import bdist_wheel root_dir = os.path.dirname(os.path.abspath(__file__)) @@ -111,10 +110,10 @@ def run(self): os.remove(f) -class BDistWheelWithStubs(_bdist_wheel): +class BDistWheelWithStubs(bdist_wheel): def run(self): build_lib = self.get_finalized_command("build_py").build_lib - taichi_path = path.join(path.dirname(path.dirname(build_lib)), "cmake-install/python") + taichi_path = os.path.join(os.path.dirname(os.path.dirname(build_lib)), "cmake-install/python") env = os.environ.copy() env["PYTHONPATH"] = taichi_path + os.pathsep + env.get("PYTHONPATH", "") @@ -125,7 +124,7 @@ def run(self): print(" ".join(cmd_line)) subprocess.check_call(cmd_line, env=env) stub_filepath = "stubs/taichi/_lib/core/taichi_python.pyi" - target_filepath = path.join(build_lib, "taichi/_lib/core/taichi_python.pyi") + target_filepath = os.path.join(build_lib, "taichi/_lib/core/taichi_python.pyi") py_typed_dst = os.path.join(build_lib, "taichi/_lib/core/py.typed") os.makedirs(os.path.dirname(target_filepath), exist_ok=True) print("copying ", stub_filepath, "to", target_filepath) diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py index 7fc2b74f91..e762f05917 100644 --- a/tests/python/test_pyi_stubs.py +++ b/tests/python/test_pyi_stubs.py @@ -1,28 +1,17 @@ -import os import subprocess import sys -import tempfile from tests import test_utils @test_utils.test() -def test_pyi_stubs(): +def test_pyi_stubs(tmpdir): test_code = """ import taichi._lib.core.taichi_python reveal_type(taichi._lib.core.taichi_python) """ - with tempfile.TemporaryDirectory() as tmpdir: - test_file = os.path.join(tmpdir, "tmp_mypy_test.py") - with open(test_file, "w") as f: - f.write(test_code) + test_file = tmpdir.join("tmp_mypy_test.py") + test_file.write(test_code) - res = subprocess.check_output( - [ - sys.executable, - "-m", - "pyright", - test_file, - ] - ).decode("utf-8") - assert "unknown" not in res.lower() + res = subprocess.check_output([sys.executable, "-m", "pyright", test_file]).decode("utf-8") + assert "unknown" not in res.lower() From b644906a660e49b293e8322aeb2586eba3359afd Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Jul 2025 07:26:28 -0700 Subject: [PATCH 65/68] / --- tests/python/test_pyi_stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py index e762f05917..93585b3663 100644 --- a/tests/python/test_pyi_stubs.py +++ b/tests/python/test_pyi_stubs.py @@ -10,7 +10,7 @@ def test_pyi_stubs(tmpdir): import taichi._lib.core.taichi_python reveal_type(taichi._lib.core.taichi_python) """ - test_file = tmpdir.join("tmp_mypy_test.py") + test_file = tmpdir / "tmp_mypy_test.py" test_file.write(test_code) res = subprocess.check_output([sys.executable, "-m", "pyright", test_file]).decode("utf-8") From d4b26feec85cc626973450cf0989043a1d7fe9c4 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Jul 2025 07:26:28 -0700 Subject: [PATCH 66/68] / --- tests/python/test_pyi_stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_pyi_stubs.py b/tests/python/test_pyi_stubs.py index e762f05917..93585b3663 100644 --- a/tests/python/test_pyi_stubs.py +++ b/tests/python/test_pyi_stubs.py @@ -10,7 +10,7 @@ def test_pyi_stubs(tmpdir): import taichi._lib.core.taichi_python reveal_type(taichi._lib.core.taichi_python) """ - test_file = tmpdir.join("tmp_mypy_test.py") + test_file = tmpdir / "tmp_mypy_test.py" test_file.write(test_code) res = subprocess.check_output([sys.executable, "-m", "pyright", test_file]).decode("utf-8") From 4fb3d606b1d94abbf98d5e60691d891022382f2c Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 2 Jul 2025 07:31:46 -0700 Subject: [PATCH 67/68] try removing contents write --- .github/workflows/api_doc.yml | 4 ++-- .github/workflows/manylinux_wheel.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index 1912d83110..762fef1e13 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -16,8 +16,8 @@ on: concurrency: group: gh-pages-deploy-${{ github.event.inputs.head_ref || github.head_ref }} cancel-in-progress: true -permissions: - contents: write +# permissions: +# contents: write jobs: deploy: runs-on: ubuntu-latest diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index d1f2e92292..cfc930b339 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -4,8 +4,8 @@ on: branches: - main workflow_dispatch: -permissions: - contents: write +# permissions: +# contents: write jobs: build_wheel: name: Manylinux wheel Build From b96a59864f173cd9f2a5dd61e8a03fec44f6b796 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 3 Jul 2025 07:26:36 -0700 Subject: [PATCH 68/68] Revert "try removing contents write" This reverts commit 4fb3d606b1d94abbf98d5e60691d891022382f2c. --- .github/workflows/api_doc.yml | 4 ++-- .github/workflows/manylinux_wheel.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/api_doc.yml b/.github/workflows/api_doc.yml index 762fef1e13..1912d83110 100644 --- a/.github/workflows/api_doc.yml +++ b/.github/workflows/api_doc.yml @@ -16,8 +16,8 @@ on: concurrency: group: gh-pages-deploy-${{ github.event.inputs.head_ref || github.head_ref }} cancel-in-progress: true -# permissions: -# contents: write +permissions: + contents: write jobs: deploy: runs-on: ubuntu-latest diff --git a/.github/workflows/manylinux_wheel.yml b/.github/workflows/manylinux_wheel.yml index cfc930b339..d1f2e92292 100644 --- a/.github/workflows/manylinux_wheel.yml +++ b/.github/workflows/manylinux_wheel.yml @@ -4,8 +4,8 @@ on: branches: - main workflow_dispatch: -# permissions: -# contents: write +permissions: + contents: write jobs: build_wheel: name: Manylinux wheel Build