From 9c30a2b34c6b1522124036e51051b30d6ddbb6e0 Mon Sep 17 00:00:00 2001 From: VinhDevNguyen Date: Fri, 19 Nov 2021 02:34:30 +0700 Subject: [PATCH 1/6] Create dockerfile and conda environment --- dockerfile | 29 +++++++++++++ environment.yml | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 dockerfile create mode 100644 environment.yml diff --git a/dockerfile b/dockerfile new file mode 100644 index 0000000..fae0cb7 --- /dev/null +++ b/dockerfile @@ -0,0 +1,29 @@ +FROM continuumio/miniconda3 + + +WORKDIR /app + +# RUN conda create -n fscount python=3.7 -y +# SHELL ["conda", "run", "-n", "fscount", "/bin/bash", "-c"] +# RUN pip install -r requirements.txt +# RUN conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.0 -c pytorch + + +# Make RUN commands use `bash --login`: +SHELL ["/bin/bash", "--login", "-c"] + +# Create the environment: +COPY environment.yml . +RUN conda env create -f environment.yml + +# Initialize conda in bash config fiiles: +RUN conda init bash + +# Fix libGL.so.1: cannot open shared object file +RUN apt-get update +RUN apt-get install ffmpeg libsm6 libxext6 -y + +# Activate the environment, and make sure it's activated: +RUN echo "conda activate fscount" > ~/.bashrc + +WORKDIR /workdir \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..1253169 --- /dev/null +++ b/environment.yml @@ -0,0 +1,112 @@ +name: fscount +channels: + - pytorch + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=4.5=1_gnu + - blas=1.0=mkl + - ca-certificates=2021.10.26=h06a4308_2 + - certifi=2021.10.8=py37h06a4308_0 + - cudatoolkit=10.0.130=0 + - freetype=2.11.0=h70c0345_0 + - giflib=5.2.1=h7b6447c_0 + - intel-openmp=2021.4.0=h06a4308_3561 + - jpeg=9d=h7f8727e_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.35.1=h7274673_9 + - libffi=3.3=he6710b0_2 + - libgcc-ng=9.3.0=h5101ec6_17 + - libgomp=9.3.0=h5101ec6_17 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.3.0=hd4cf53a_17 + - libtiff=4.2.0=h85742a9_0 + - libwebp=1.2.0=h89dd481_0 + - libwebp-base=1.2.0=h27cfd23_0 + - lz4-c=1.9.3=h295c915_1 + - mkl=2021.4.0=h06a4308_640 + - mkl-service=2.4.0=py37h7f8727e_0 + - mkl_fft=1.3.1=py37hd3c417c_0 + - mkl_random=1.2.2=py37h51133e4_0 + - ncurses=6.3=h7f8727e_2 + - ninja=1.10.2=py37hd09550d_3 + - numpy-base=1.21.2=py37h79a1101_0 + - olefile=0.46=py37_0 + - openssl=1.1.1l=h7f8727e_0 + - pip=21.0.1=py37h06a4308_0 + - python=3.7.11=h12debd9_0 + - pytorch=1.4.0=py3.7_cuda10.0.130_cudnn7.6.3_0 + - readline=8.1=h27cfd23_0 + - setuptools=58.0.4=py37h06a4308_0 + - six=1.16.0=pyhd3eb1b0_0 + - sqlite=3.36.0=hc218d9a_0 + - tk=8.6.11=h1ccaba5_0 + - torchvision=0.5.0=py37_cu100 + - wheel=0.37.0=pyhd3eb1b0_1 + - xz=5.2.5=h7b6447c_0 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.4.9=haebb681_0 + - pip: + - argcomplete==1.12.3 + - argon2-cffi==21.1.0 + - attrs==21.2.0 + - backcall==0.2.0 + - bleach==4.1.0 + - cffi==1.15.0 + - cycler==0.11.0 + - debugpy==1.5.1 + - decorator==5.1.0 + - defusedxml==0.7.1 + - entrypoints==0.3 + - fonttools==4.28.1 + - importlib-metadata==4.8.2 + - importlib-resources==5.4.0 + - ipykernel==6.5.0 + - ipython==7.29.0 + - ipython-genutils==0.2.0 + - jedi==0.18.0 + - jinja2==3.0.3 + - jsonschema==4.2.1 + - jupyter-client==7.0.6 + - jupyter-core==4.9.1 + - jupyterlab-pygments==0.1.2 + - kiwisolver==1.3.2 + - markupsafe==2.0.1 + - matplotlib==3.5.0 + - matplotlib-inline==0.1.3 + - mistune==0.8.4 + - nbclient==0.5.8 + - nbconvert==6.3.0 + - nbformat==5.1.3 + - nest-asyncio==1.5.1 + - notebook==6.4.5 + - numpy==1.21.4 + - opencv-python==4.5.4.58 + - packaging==21.2 + - pandocfilters==1.5.0 + - parso==0.8.2 + - pexpect==4.8.0 + - pickleshare==0.7.5 + - pillow==8.4.0 + - prometheus-client==0.12.0 + - prompt-toolkit==3.0.22 + - ptyprocess==0.7.0 + - pycparser==2.21 + - pygments==2.10.0 + - pyparsing==2.4.7 + - pyrsistent==0.18.0 + - python-dateutil==2.8.2 + - pyzmq==22.3.0 + - send2trash==1.8.0 + - setuptools-scm==6.3.2 + - terminado==0.12.1 + - testpath==0.5.0 + - tomli==1.2.2 + - tornado==6.1 + - tqdm==4.62.3 + - traitlets==5.1.1 + - typing-extensions==4.0.0 + - wcwidth==0.2.5 + - webencodings==0.5.1 + - zipp==3.6.0 +prefix: /opt/conda/envs/fscount From dc5f23e2f9a1455a2386a7714bf904471fc1b9e6 Mon Sep 17 00:00:00 2001 From: VinhDevNguyen Date: Fri, 19 Nov 2021 03:10:09 +0700 Subject: [PATCH 2/6] Update docker instruction --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index dbda86e..070c2c3 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ python -m pip install matplotlib opencv-python notebook tqdm conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.0 -c pytorch +## Run with docker file +- Build dockerfile `docker build -t cvlab/cvcount .` +- Run image `docker run --rm --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=0,1 -ti -v $PWD:/workdir cvlab/cvcount` +- Type `bash --login` in docker terminal ## Quick demo From e66e88e10e1ac31e724d94edcd8501b5eb0f745e Mon Sep 17 00:00:00 2001 From: VinhDevNguyen Date: Fri, 19 Nov 2021 03:13:44 +0700 Subject: [PATCH 3/6] Fix docker not recognize GPU --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 070c2c3..6e9abd6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.0 -c pytorch ## Run with docker file - Build dockerfile `docker build -t cvlab/cvcount .` -- Run image `docker run --rm --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=0,1 -ti -v $PWD:/workdir cvlab/cvcount` +- Run image `docker run --rm --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=0,1 -e NVIDIA_DRIVER_CAPABILITIES=compute,utility -ti -v $PWD:/workdir cvlab/cvcount` - Type `bash --login` in docker terminal ## Quick demo From fcd720a567eb0edc3dab83809d76ffaa7e903e76 Mon Sep 17 00:00:00 2001 From: VinhDevNguyen Date: Fri, 19 Nov 2021 14:24:13 +0700 Subject: [PATCH 4/6] Change default data_path in train.py --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 2273f0f..dd8e355 100644 --- a/train.py +++ b/train.py @@ -22,7 +22,7 @@ parser = argparse.ArgumentParser(description="Few Shot Counting Evaluation code") -parser.add_argument("-dp", "--data_path", type=str, default='/home/hoai/DataSets/AgnosticCounting/FSC147_384_V2/', help="Path to the FSC147 dataset") +parser.add_argument("-dp", "--data_path", type=str, default='./data', help="Path to the FSC147 dataset") parser.add_argument("-o", "--output_dir", type=str,default="./logsSave", help="/Path/to/output/logs/") parser.add_argument("-ts", "--test-split", type=str, default='val', choices=["train", "test", "val"], help="what data split to evaluate on on") parser.add_argument("-ep", "--epochs", type=int,default=1500, help="number of training epochs") From 9567d4520697740e8c7f484195e1f70d5090b5b8 Mon Sep 17 00:00:00 2001 From: VinhDevNguyen Date: Fri, 19 Nov 2021 14:26:46 +0700 Subject: [PATCH 5/6] Fix data_path --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index dd8e355..652e5a6 100644 --- a/train.py +++ b/train.py @@ -22,7 +22,7 @@ parser = argparse.ArgumentParser(description="Few Shot Counting Evaluation code") -parser.add_argument("-dp", "--data_path", type=str, default='./data', help="Path to the FSC147 dataset") +parser.add_argument("-dp", "--data_path", type=str, default='./data/', help="Path to the FSC147 dataset") parser.add_argument("-o", "--output_dir", type=str,default="./logsSave", help="/Path/to/output/logs/") parser.add_argument("-ts", "--test-split", type=str, default='val', choices=["train", "test", "val"], help="what data split to evaluate on on") parser.add_argument("-ep", "--epochs", type=int,default=1500, help="number of training epochs") From 4f45389f718516a4877ab3a3c4bd5811925df3e6 Mon Sep 17 00:00:00 2001 From: VinhDevNguyen Date: Fri, 19 Nov 2021 15:12:22 +0700 Subject: [PATCH 6/6] Update wandb for monitor purpose --- environment.yml | 1 + train.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 1253169..c960b70 100644 --- a/environment.yml +++ b/environment.yml @@ -109,4 +109,5 @@ dependencies: - wcwidth==0.2.5 - webencodings==0.5.1 - zipp==3.6.0 + - wandb prefix: /opt/conda/envs/fscount diff --git a/train.py b/train.py index 652e5a6..21d4a6a 100644 --- a/train.py +++ b/train.py @@ -19,6 +19,9 @@ import random import torch.optim as optim import torch.nn.functional as F +import wandb + +wandb.init(project="learning-to-count-every-thing") parser = argparse.ArgumentParser(description="Few Shot Counting Evaluation code") @@ -184,7 +187,15 @@ def eval(): best_rmse = val_rmse model_name = args.output_dir + '/' + "FamNet.pth" torch.save(regressor.state_dict(), model_name) - + # Log loss + wandb.log({"Epoch": epoch+1, + "Avg. Epoch Loss:": stats[-1][0], + "Train MAE": stats[-1][1], + "Train RMSE": stats[-1][2], + "Val MAE": stats[-1][3], + "Val RMSE": stats[-1][4] + }) + print("") print("Epoch {}, Avg. Epoch Loss: {} Train MAE: {} Train RMSE: {} Val MAE: {} Val RMSE: {} Best Val MAE: {} Best Val RMSE: {} ".format( epoch+1, stats[-1][0], stats[-1][1], stats[-1][2], stats[-1][3], stats[-1][4], best_mae, best_rmse))