From 30c3aaabf610c5e67ec09db9b73a8f2a407845c1 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 03:18:41 +0530 Subject: [PATCH 01/27] udpate --- frontend/dashboard.py | 49 ------------------------- test.py | 5 +++ yoloexplorer/explorer.py | 8 +++- yoloexplorer/frontend/__init__.py | 3 ++ yoloexplorer/frontend/launch.py | 61 +++++++++++++++++++++++++++++++ yoloexplorer/frontend/layout.py | 35 ++++++++++++++++++ yoloexplorer/frontend/states.py | 8 ++++ 7 files changed, 119 insertions(+), 50 deletions(-) delete mode 100644 frontend/dashboard.py create mode 100644 test.py create mode 100644 yoloexplorer/frontend/__init__.py create mode 100644 yoloexplorer/frontend/launch.py create mode 100644 yoloexplorer/frontend/layout.py create mode 100644 yoloexplorer/frontend/states.py diff --git a/frontend/dashboard.py b/frontend/dashboard.py deleted file mode 100644 index 5f80277..0000000 --- a/frontend/dashboard.py +++ /dev/null @@ -1,49 +0,0 @@ -import streamlit as st -from yoloexplorer import Explorer -from streamlit_image_select import image_select -from typing import List - -st.title("YOLOExplorer") - -with st.sidebar: - default_data_yaml = "coco128.yaml" - st.text(f"Dataset: {default_data_yaml}") - if default_data_yaml: - if 'data_yaml' not in st.session_state: - st.session_state['data_yaml'] = default_data_yaml - if 'session' not in st.session_state: - st.session_state['session'] = Explorer(default_data_yaml, model=None) - dataset_info = st.session_state['session'].dataset_info - - st.write("**Dataset Information :**") - - for dataset_type in ["train", "val", "tets"]: - st.text(f"{dataset_type.capitalize()} paths :") - if isinstance(dataset_info['train'], str): - st.text(dataset_info['train']) - elif isinstance(dataset_info['train', List]): - for p in dataset_info['train']: - st.text(p) - -st.write("**Dataset Visualization**") - -# TODO: Only show one of the sample dataset at same time -img_train_checkbox_w, img_val_checkbox_w, img_test_checkbox_w, num_sample_w = st.columns(4) -with img_train_checkbox_w: - img_train_checkbox = st.checkbox(label="Train") -with img_val_checkbox_w: - img_val_checkbox = st.checkbox(label="Val") -with img_test_checkbox_w: - img_test_checkbox = st.checkbox(label="Test") -with num_sample_w: - num_samples_show = st.number_input(label="Numbers of Images", value=10) - - -if num_samples_show: - if img_train_checkbox: - img_url = "/home/hd/zidane.jpg" # Put path to your image - train_image_paths = [img_url] * int(num_samples_show) - # st.image(train_image_paths, use_column_width="always") - clicked = image_select("Training Samples", images=train_image_paths) - - diff --git a/test.py b/test.py new file mode 100644 index 0000000..421d1e2 --- /dev/null +++ b/test.py @@ -0,0 +1,5 @@ +from yoloexplorer import Explorer + +exp = Explorer("coco128.yaml") +exp.build_embeddings() +exp.dash() diff --git a/yoloexplorer/explorer.py b/yoloexplorer/explorer.py index 2ed6cdc..5241ce9 100644 --- a/yoloexplorer/explorer.py +++ b/yoloexplorer/explorer.py @@ -21,7 +21,7 @@ from yoloexplorer.dataset import get_dataset_info, Dataset from yoloexplorer.yolo_predictor import YOLOEmbeddingsPredictor - +from yoloexplorer.frontend import dash SCHEMA = [ "id", @@ -487,6 +487,12 @@ def sql(self, query: str): return result + def dash(self): + """ + Launches a dashboard to visualize the dataset. + """ + dash(self) + def _log_training_cmd(self, data_path): LOGGER.info( f'{colorstr("LanceDB: ") }New dataset created successfully! Run the following command to train a model:' diff --git a/yoloexplorer/frontend/__init__.py b/yoloexplorer/frontend/__init__.py new file mode 100644 index 0000000..8840c59 --- /dev/null +++ b/yoloexplorer/frontend/__init__.py @@ -0,0 +1,3 @@ +from .layout import dash + +__all__ = ["dash"] \ No newline at end of file diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py new file mode 100644 index 0000000..a1f9ef8 --- /dev/null +++ b/yoloexplorer/frontend/launch.py @@ -0,0 +1,61 @@ +from collections import defaultdict as _defaultdict +from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator +from typing import Dict as _Dict +import streamlit as _st +import sys as _sys + + +# EXPORT +st = _st +ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) + + +# VARS +_IS_WRAPPED = False + + +def run_streamlit(python_file): + """ + IMPORTANT: import this function from a seperate file! + Use this function by placing a call to it at the top of your + check for __main__ + >>> from run_streamlit import run_streamlit, st, ST + >>> + >>> if __name__ == '__main__': + >>> run_streamlit(__file__) + >>> st.title('Hello') + >>> ST['info'].text('World!') + """ + + # do not wrap if streamlit is already running! + global _IS_WRAPPED + if _IS_WRAPPED: + return True + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + import click + import streamlit.cli + + @click.group() + def run_streamlit(): + pass + + # For some reason I cant get streamlit to work without this subcommand? + @run_streamlit.command('streamlit') + @streamlit.cli.configurator_options + def run_streamlit_subcommand(**kwargs): + global _IS_WRAPPED + _IS_WRAPPED = True + streamlit.cli._apply_config_options_from_cli(kwargs) + streamlit.cli._main_run(python_file, args) + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # swap out arguments and keep arguments, first is the filename + args = _sys.argv[1:] + _sys.argv = [_sys.argv[0], 'streamlit'] + + # run streamlit + run_streamlit() + exit(0) \ No newline at end of file diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py new file mode 100644 index 0000000..9550571 --- /dev/null +++ b/yoloexplorer/frontend/layout.py @@ -0,0 +1,35 @@ +import streamlit as st +from streamlit_image_select import image_select +from typing import List + +from .launch import run_streamlit +from .states import init_states, update_state + +def layout(): + st.title("YOLOExplorer") + + with st.sidebar: + dataset_info = st.session_state.EXPLORER.get_dataset_info() + st.write("**Dataset Information :**") + + for dataset_type in ["train", "val", "tets"]: + st.text(f"{dataset_type.capitalize()} paths :") + if isinstance(dataset_info['train'], str): + st.text(dataset_info['train']) + elif isinstance(dataset_info['train', List]): + for p in dataset_info['train']: + st.text(p) + + st.write("**Dataset Visualization**") + + exp = st.session_state.EXPLORER + st.session_state.EXPLORER.IMGS = exp.to_pandas()["path"].to_list() + if st.session_state.EXPLORER.IMGS: + clicked = image_select("Samples", images=st.session_state.EXPLORER.IMGS) + + +def dash(explorer): + run_streamlit(__file__) + init_states() + update_state("EXPLORER", explorer) + layout() \ No newline at end of file diff --git a/yoloexplorer/frontend/states.py b/yoloexplorer/frontend/states.py new file mode 100644 index 0000000..53570a8 --- /dev/null +++ b/yoloexplorer/frontend/states.py @@ -0,0 +1,8 @@ +import streamlit as st + +def init_states(): + st.session.EXPLORER = None + st.session_state.IMGS = [] + +def update_state(state, value): + st.session_state[state] = value \ No newline at end of file From d19df58410d931eeb76cc6caa43c646d8c217ca6 Mon Sep 17 00:00:00 2001 From: Onuralp SEZER Date: Fri, 14 Jul 2023 07:09:47 +0300 Subject: [PATCH 02/27] =?UTF-8?q?chore(gitignore):=20=F0=9F=A7=B9=20update?= =?UTF-8?q?=20gitignore=20and=20clean=20up=20egg=20folder=20from=20reposit?= =?UTF-8?q?ory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Onuralp SEZER --- .gitignore | 366 ++++++++++++++++++++++++++++- yoloexplorer.egg-info/requires.txt | 4 - 2 files changed, 357 insertions(+), 13 deletions(-) delete mode 100644 yoloexplorer.egg-info/requires.txt diff --git a/.gitignore b/.gitignore index a840fa3..19134e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,358 @@ -.pytest_cache -env/ -run/ -*.pt -__pycache__/ -.ipynb_checkpoints/ -*_updated.yaml -*_updated.txt +### JetBrains ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General .DS_Store -*.egg-info/* \ No newline at end of file +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk diff --git a/yoloexplorer.egg-info/requires.txt b/yoloexplorer.egg-info/requires.txt deleted file mode 100644 index 0748370..0000000 --- a/yoloexplorer.egg-info/requires.txt +++ /dev/null @@ -1,4 +0,0 @@ -lancedb -duckdb -scikit-learn -ultralytics From 908640d854adf76e73006b887a28ad54f9d649fc Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 11:31:39 +0530 Subject: [PATCH 03/27] Update README.md --- README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/README.md b/README.md index fa4668e..6b4498d 100644 --- a/README.md +++ b/README.md @@ -176,12 +176,3 @@ Notes: * The API will have some minor changes going from dev to minor release * For all practical purposes the ids are same as row number and is reset after every addition or removal - - -## Dashboard: - -Start dashboard application using following command. -``` -streamlit run frontend/dashboard.py --server.port 8000 -``` - From ba59c6d0a50bd7b134f50dccf6b5d9705b49b157 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 22:30:07 +0530 Subject: [PATCH 04/27] update --- yoloexplorer/explorer.py | 25 +++++++++++++++++++-- yoloexplorer/frontend/__init__.py | 4 ++-- yoloexplorer/frontend/layout.py | 37 ++++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/yoloexplorer/explorer.py b/yoloexplorer/explorer.py index 5241ce9..99d057c 100644 --- a/yoloexplorer/explorer.py +++ b/yoloexplorer/explorer.py @@ -1,6 +1,8 @@ from pathlib import Path from collections import defaultdict import math +import json +import tempfile import pandas as pd import cv2 @@ -21,7 +23,7 @@ from yoloexplorer.dataset import get_dataset_info, Dataset from yoloexplorer.yolo_predictor import YOLOEmbeddingsPredictor -from yoloexplorer.frontend import dash +from yoloexplorer.frontend import launch SCHEMA = [ "id", @@ -68,6 +70,7 @@ def __init__(self, data, model="yolov8n.pt", device="", project="run") -> None: self.data = data self.table = None self.model = model + self.device = device self.project = project self.dataset_info = None self.predictor = None @@ -491,7 +494,25 @@ def dash(self): """ Launches a dashboard to visualize the dataset. """ - dash(self) + (Path(self.project) / 'config').mkdir(exist_ok=True) + with tempfile.NamedTemporaryFile(prefix='_temp_', + suffix=f'{id(self)}.json', + mode='w+', + encoding='utf-8', + dir=Path(self.project) / 'config', + delete=False) as file: + json.dump(self.config, file) + + launch(file) + + @property + def config(self): + return { + "project": self.project, + "model": self.model, + "device": self.device, + "data": self.data + } def _log_training_cmd(self, data_path): LOGGER.info( diff --git a/yoloexplorer/frontend/__init__.py b/yoloexplorer/frontend/__init__.py index 8840c59..81634c2 100644 --- a/yoloexplorer/frontend/__init__.py +++ b/yoloexplorer/frontend/__init__.py @@ -1,3 +1,3 @@ -from .layout import dash +from .layout import launch -__all__ = ["dash"] \ No newline at end of file +__all__ = ["launch"] \ No newline at end of file diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 9550571..66473b6 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -1,11 +1,24 @@ +import sys +import json +import argparse import streamlit as st from streamlit_image_select import image_select from typing import List +import streamlit.web.cli as stcli -from .launch import run_streamlit -from .states import init_states, update_state +from yoloexplorer.frontend.states import init_states, update_state -def layout(): +def argparser(): + parser = argparse.ArgumentParser() + parser.add_argument("--info", type=str, required=True) + return parser.parse_args() + +def layout(info): + # function scope import + from yoloexplorer import Explorer + import pdb; pdb.set_trace() + init_states() + update_state("EXPLORER", Explorer(**json.load(info))) st.title("YOLOExplorer") with st.sidebar: @@ -23,13 +36,15 @@ def layout(): st.write("**Dataset Visualization**") exp = st.session_state.EXPLORER - st.session_state.EXPLORER.IMGS = exp.to_pandas()["path"].to_list() - if st.session_state.EXPLORER.IMGS: - clicked = image_select("Samples", images=st.session_state.EXPLORER.IMGS) + st.session_state.IMGS = exp.to_pandas()["path"].to_list() + if st.session_state.IMGS: + clicked = image_select("Samples", images=st.session_state.IMGS) -def dash(explorer): - run_streamlit(__file__) - init_states() - update_state("EXPLORER", explorer) - layout() \ No newline at end of file +def launch(info): + sys.argv = ["streamlit", "run", __file__, "-- --info", info] + stcli.main() + +if __name__ == "__main__": + args = argparser() + layout(args.info) \ No newline at end of file From 1bcad0e8413879bc69fbccc8f889770869f3a0fa Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 23:59:18 +0530 Subject: [PATCH 05/27] update --- yoloexplorer/__init__.py | 6 ++-- yoloexplorer/frontend/launch.py | 61 --------------------------------- yoloexplorer/frontend/layout.py | 29 ++++++++++++---- yoloexplorer/frontend/states.py | 6 ---- 4 files changed, 25 insertions(+), 77 deletions(-) delete mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index c136bcf..3f7404f 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,3 @@ -from .explorer import Explorer - -__all__ = ["Explorer"] +from .explorer import Explorer +from .frontend import layout +__all__ = ["Explorer", "layout"] diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py deleted file mode 100644 index a1f9ef8..0000000 --- a/yoloexplorer/frontend/launch.py +++ /dev/null @@ -1,61 +0,0 @@ -from collections import defaultdict as _defaultdict -from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator -from typing import Dict as _Dict -import streamlit as _st -import sys as _sys - - -# EXPORT -st = _st -ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) - - -# VARS -_IS_WRAPPED = False - - -def run_streamlit(python_file): - """ - IMPORTANT: import this function from a seperate file! - Use this function by placing a call to it at the top of your - check for __main__ - >>> from run_streamlit import run_streamlit, st, ST - >>> - >>> if __name__ == '__main__': - >>> run_streamlit(__file__) - >>> st.title('Hello') - >>> ST['info'].text('World!') - """ - - # do not wrap if streamlit is already running! - global _IS_WRAPPED - if _IS_WRAPPED: - return True - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - import click - import streamlit.cli - - @click.group() - def run_streamlit(): - pass - - # For some reason I cant get streamlit to work without this subcommand? - @run_streamlit.command('streamlit') - @streamlit.cli.configurator_options - def run_streamlit_subcommand(**kwargs): - global _IS_WRAPPED - _IS_WRAPPED = True - streamlit.cli._apply_config_options_from_cli(kwargs) - streamlit.cli._main_run(python_file, args) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # swap out arguments and keep arguments, first is the filename - args = _sys.argv[1:] - _sys.argv = [_sys.argv[0], 'streamlit'] - - # run streamlit - run_streamlit() - exit(0) \ No newline at end of file diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 66473b6..d4bef19 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -1,12 +1,18 @@ import sys import json +import subprocess import argparse import streamlit as st from streamlit_image_select import image_select from typing import List import streamlit.web.cli as stcli -from yoloexplorer.frontend.states import init_states, update_state +def init_states(): + st.session_state.EXPLORER = None + st.session_state.IMGS = [] + +def update_state(state, value): + st.session_state[state] = value def argparser(): parser = argparse.ArgumentParser() @@ -16,13 +22,17 @@ def argparser(): def layout(info): # function scope import from yoloexplorer import Explorer - import pdb; pdb.set_trace() + init_states() - update_state("EXPLORER", Explorer(**json.load(info))) + with open(info) as json_file: + data = json.load(json_file) + exp = Explorer(**data) + exp.build_embeddings() + update_state("EXPLORER", exp) st.title("YOLOExplorer") with st.sidebar: - dataset_info = st.session_state.EXPLORER.get_dataset_info() + dataset_info = st.session_state.EXPLORER.dataset_info st.write("**Dataset Information :**") for dataset_type in ["train", "val", "tets"]: @@ -36,14 +46,19 @@ def layout(info): st.write("**Dataset Visualization**") exp = st.session_state.EXPLORER - st.session_state.IMGS = exp.to_pandas()["path"].to_list() + st.session_state.IMGS = exp.table.to_pandas()["path"].to_list() if st.session_state.IMGS: clicked = image_select("Samples", images=st.session_state.IMGS) def launch(info): - sys.argv = ["streamlit", "run", __file__, "-- --info", info] - stcli.main() + cmd = ["streamlit", "run", __file__, "-- --info", str(info.name)] + try: + subprocess.run(cmd, check=True) + except Exception as e: + print(e) + + #stcli.main() if __name__ == "__main__": args = argparser() diff --git a/yoloexplorer/frontend/states.py b/yoloexplorer/frontend/states.py index 53570a8..1f0556a 100644 --- a/yoloexplorer/frontend/states.py +++ b/yoloexplorer/frontend/states.py @@ -1,8 +1,2 @@ import streamlit as st -def init_states(): - st.session.EXPLORER = None - st.session_state.IMGS = [] - -def update_state(state, value): - st.session_state[state] = value \ No newline at end of file From 5aa2de8a037954c3dbb89d8810cadac36e592f53 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 00:38:26 +0530 Subject: [PATCH 06/27] update --- .gitignore | 1 + yoloexplorer/__init__.py | 3 ++- yoloexplorer/config.py | 1 + yoloexplorer/explorer.py | 12 ++++-------- yoloexplorer/frontend/layout.py | 19 ++++++------------- 5 files changed, 14 insertions(+), 22 deletions(-) create mode 100644 yoloexplorer/config.py diff --git a/.gitignore b/.gitignore index a840fa3..ee67489 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ __pycache__/ *_updated.yaml *_updated.txt .DS_Store +.config *.egg-info/* \ No newline at end of file diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index 3f7404f..460bde0 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,4 @@ from .explorer import Explorer from .frontend import layout -__all__ = ["Explorer", "layout"] +from . import config +__all__ = ["Explorer", "layout", "config"] diff --git a/yoloexplorer/config.py b/yoloexplorer/config.py new file mode 100644 index 0000000..509ca19 --- /dev/null +++ b/yoloexplorer/config.py @@ -0,0 +1 @@ +TEMP_CONFIG_PATH = ".config/last_config.json" \ No newline at end of file diff --git a/yoloexplorer/explorer.py b/yoloexplorer/explorer.py index 99d057c..c18f6bd 100644 --- a/yoloexplorer/explorer.py +++ b/yoloexplorer/explorer.py @@ -24,6 +24,7 @@ from yoloexplorer.dataset import get_dataset_info, Dataset from yoloexplorer.yolo_predictor import YOLOEmbeddingsPredictor from yoloexplorer.frontend import launch +from yoloexplorer.config import TEMP_CONFIG_PATH SCHEMA = [ "id", @@ -494,16 +495,11 @@ def dash(self): """ Launches a dashboard to visualize the dataset. """ - (Path(self.project) / 'config').mkdir(exist_ok=True) - with tempfile.NamedTemporaryFile(prefix='_temp_', - suffix=f'{id(self)}.json', - mode='w+', - encoding='utf-8', - dir=Path(self.project) / 'config', - delete=False) as file: + Path(TEMP_CONFIG_PATH).parent.mkdir(exist_ok=True, parents=True) + with open(TEMP_CONFIG_PATH, "w+") as file: json.dump(self.config, file) - launch(file) + launch() @property def config(self): diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index d4bef19..8a7856c 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -6,6 +6,7 @@ from streamlit_image_select import image_select from typing import List import streamlit.web.cli as stcli +from yoloexplorer import config def init_states(): st.session_state.EXPLORER = None @@ -14,17 +15,12 @@ def init_states(): def update_state(state, value): st.session_state[state] = value -def argparser(): - parser = argparse.ArgumentParser() - parser.add_argument("--info", type=str, required=True) - return parser.parse_args() - -def layout(info): +def layout(): # function scope import from yoloexplorer import Explorer init_states() - with open(info) as json_file: + with open(config.TEMP_CONFIG_PATH) as json_file: data = json.load(json_file) exp = Explorer(**data) exp.build_embeddings() @@ -51,15 +47,12 @@ def layout(info): clicked = image_select("Samples", images=st.session_state.IMGS) -def launch(info): - cmd = ["streamlit", "run", __file__, "-- --info", str(info.name)] +def launch(): + cmd = ["streamlit", "run", __file__] try: subprocess.run(cmd, check=True) except Exception as e: print(e) - - #stcli.main() if __name__ == "__main__": - args = argparser() - layout(args.info) \ No newline at end of file + layout() \ No newline at end of file From a300ea43925687940e72ec6cab6a567ec59e251f Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 00:43:51 +0530 Subject: [PATCH 07/27] update --- yoloexplorer/explorer.py | 1 - yoloexplorer/frontend/layout.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/yoloexplorer/explorer.py b/yoloexplorer/explorer.py index c18f6bd..86f6c95 100644 --- a/yoloexplorer/explorer.py +++ b/yoloexplorer/explorer.py @@ -2,7 +2,6 @@ from collections import defaultdict import math import json -import tempfile import pandas as pd import cv2 diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 8a7856c..a6374af 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -1,11 +1,8 @@ -import sys import json import subprocess -import argparse import streamlit as st from streamlit_image_select import image_select from typing import List -import streamlit.web.cli as stcli from yoloexplorer import config def init_states(): @@ -44,7 +41,7 @@ def layout(): exp = st.session_state.EXPLORER st.session_state.IMGS = exp.table.to_pandas()["path"].to_list() if st.session_state.IMGS: - clicked = image_select("Samples", images=st.session_state.IMGS) + clicked = image_select("Samples", images=st.session_state.IMGS) #noqa def launch(): From 8d752dd2d59209d1f2cb1f779441aa0d65651491 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 18:28:28 +0530 Subject: [PATCH 08/27] update --- yoloexplorer/frontend/layout.py | 25 +++++-------------------- yoloexplorer/frontend/states.py | 6 ++++++ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index a6374af..8fc1214 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -4,13 +4,7 @@ from streamlit_image_select import image_select from typing import List from yoloexplorer import config - -def init_states(): - st.session_state.EXPLORER = None - st.session_state.IMGS = [] - -def update_state(state, value): - st.session_state[state] = value +from yoloexplorer.frontend.states import init_states, update_state def layout(): # function scope import @@ -24,28 +18,19 @@ def layout(): update_state("EXPLORER", exp) st.title("YOLOExplorer") - with st.sidebar: - dataset_info = st.session_state.EXPLORER.dataset_info - st.write("**Dataset Information :**") - - for dataset_type in ["train", "val", "tets"]: - st.text(f"{dataset_type.capitalize()} paths :") - if isinstance(dataset_info['train'], str): - st.text(dataset_info['train']) - elif isinstance(dataset_info['train', List]): - for p in dataset_info['train']: - st.text(p) st.write("**Dataset Visualization**") exp = st.session_state.EXPLORER st.session_state.IMGS = exp.table.to_pandas()["path"].to_list() if st.session_state.IMGS: - clicked = image_select("Samples", images=st.session_state.IMGS) #noqa + num = min(4000, len(st.session_state.IMGS)) + + clicked = image_select("Samples", images=st.session_state.IMGS[0:num]) #noqa def launch(): - cmd = ["streamlit", "run", __file__] + cmd = ["streamlit", "run", __file__, "--server.maxMessageSize", "1024"] try: subprocess.run(cmd, check=True) except Exception as e: diff --git a/yoloexplorer/frontend/states.py b/yoloexplorer/frontend/states.py index 1f0556a..ec37164 100644 --- a/yoloexplorer/frontend/states.py +++ b/yoloexplorer/frontend/states.py @@ -1,2 +1,8 @@ import streamlit as st +def init_states(): + st.session_state.EXPLORER = None + st.session_state.IMGS = [] + +def update_state(state, value): + st.session_state[state] = value \ No newline at end of file From 28ce670c6db4de771179313349ab96545513919c Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 18:37:11 +0530 Subject: [PATCH 09/27] update --- yoloexplorer/frontend/layout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 8fc1214..508f001 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -2,7 +2,6 @@ import subprocess import streamlit as st from streamlit_image_select import image_select -from typing import List from yoloexplorer import config from yoloexplorer.frontend.states import init_states, update_state From 818c89b6a7bd1292726855906849b01d8c4b78d7 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 20:32:33 +0530 Subject: [PATCH 10/27] update --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index d0d93af..47a94e8 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,8 @@ "lancedb", "duckdb", "scikit-learn", +"streamlit", +"streamlit-image-select" "ultralytics@git+https://github.com/ultralytics/ultralytics.git@embeddings" ] From 14f107b228189ac33ea1107b1c3c40a26743ee6f Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 20:38:51 +0530 Subject: [PATCH 11/27] update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47a94e8..d325781 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ "duckdb", "scikit-learn", "streamlit", -"streamlit-image-select" +"streamlit-image-select", "ultralytics@git+https://github.com/ultralytics/ultralytics.git@embeddings" ] From a556b2c5edeca2c37a023b5e3e5051215b92f545 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 17 Jul 2023 17:40:50 +0530 Subject: [PATCH 12/27] update --- tests/test_explorer.py | 6 ++ yoloexplorer/explorer.py | 11 +++- yoloexplorer/frontend/layout.py | 80 ++++++++++++++++++++---- yoloexplorer/frontend/pages/embedding.py | 0 yoloexplorer/frontend/states.py | 4 ++ 5 files changed, 86 insertions(+), 15 deletions(-) create mode 100644 yoloexplorer/frontend/pages/embedding.py diff --git a/tests/test_explorer.py b/tests/test_explorer.py index 5a7376a..6e7bab9 100644 --- a/tests/test_explorer.py +++ b/tests/test_explorer.py @@ -83,6 +83,12 @@ def test_id_reassignment(self): df = coco_exp.table.to_pandas() assert df["id"].to_list() == [idx for idx in range(128)], "the ids should be reassigned" + def test_sim_search(): + coco_exp = Explorer("coco128.yaml") + coco_exp.build_embeddings() + coco_exp.get_similar_imgs(0, 10) + coco_exp.get_similar_imgs([0,1], 10) + """ # Not supported yet def test_copy_embeddings_from_table(self): diff --git a/yoloexplorer/explorer.py b/yoloexplorer/explorer.py index 86f6c95..c86d1cf 100644 --- a/yoloexplorer/explorer.py +++ b/yoloexplorer/explorer.py @@ -10,7 +10,7 @@ import numpy as np import yaml from tqdm import tqdm - +from typing import List from ultralytics import YOLO from ultralytics.yolo.utils import LOGGER, colorstr from ultralytics.yolo.utils.plotting import Annotator, colors @@ -201,12 +201,19 @@ def get_similar_imgs(self, img, n=10): img = img elif isinstance(img, bytes): img = decode(img) + elif isinstance(img, list): # exceptional case for batch search from dash + df = self.table.to_pandas().set_index("path") + array = df.loc[img]["vector"].to_list() + embeddings = np.array(array) + if len(embeddings) > 1: + embeddings = np.mean(embeddings, axis=0) + else: + embeddings = np.squeeze(embeddings) else: LOGGER.error( "img should be index from the table(int) or path of an image (str or Path)" ) return - if embeddings is None: embeddings = self.predictor.embed(img).squeeze().cpu().numpy() sim = self.table.search(embeddings).limit(n).to_df() diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 508f001..3e34521 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -1,31 +1,85 @@ import json import subprocess + import streamlit as st -from streamlit_image_select import image_select +from streamlit_dash import image_select from yoloexplorer import config -from yoloexplorer.frontend.states import init_states, update_state +from yoloexplorer.frontend.states import init_states, update_state, INDEX_PAGE_QUERY_FORM_KEY, INDEX_PAGE_SIMILARITY_FORM_KEY -def layout(): - # function scope import - from yoloexplorer import Explorer +@st.cache_data +def _get_dataset(): + from yoloexplorer import Explorer # function scope import - init_states() with open(config.TEMP_CONFIG_PATH) as json_file: data = json.load(json_file) exp = Explorer(**data) exp.build_embeddings() - update_state("EXPLORER", exp) - st.title("YOLOExplorer") + return exp + +def reset_to_init_state(): + if st.session_state.get("EXPLORER") is None: + init_states() + exp = _get_dataset() + update_state("EXPLORER", exp) + update_state("IMGS", exp.table.to_pandas()["path"].to_list()) + +def query_form(): + with st.form(INDEX_PAGE_QUERY_FORM_KEY): + col1, col2 = st.columns([0.8, 0.2]) + with col1: + query = st.text_input("Query", "", label_visibility="collapsed") + with col2: + submitted = st.form_submit_button("Query") + if submitted: + if query: + exp = st.session_state.EXPLORER + df = exp.sql(query) + update_state("IMGS", df["path"].to_list()) + +def similarity_form(selected_imgs): + st.write("Similarity Search") + with st.form(INDEX_PAGE_SIMILARITY_FORM_KEY): + subcol1, subcol2 = st.columns([1,1]) + with subcol1: + st.write("Limit") + limit = st.number_input("limit", min_value=None, max_value=None, value=25, label_visibility="collapsed") + + with subcol2: + st.write("Selected: ", len(selected_imgs)) + submitted = st.form_submit_button("Search") - st.write("**Dataset Visualization**") + if submitted: + find_similar_imgs(selected_imgs, limit=limit) +def find_similar_imgs(imgs, limit=25): exp = st.session_state.EXPLORER - st.session_state.IMGS = exp.table.to_pandas()["path"].to_list() - if st.session_state.IMGS: - num = min(4000, len(st.session_state.IMGS)) + df = exp.table.to_pandas() + _, idx = exp.get_similar_imgs(imgs, limit) + paths = df["path"][idx].to_list() + update_state("IMGS", paths) + st.experimental_rerun() + print("updated IMGS") + + + +def layout(): + st.set_page_config(layout='wide') + col1, col2= st.columns([0.75, 0.25], gap="small") + + reset_to_init_state() + with col1: + subcol1, subcol2 = st.columns([0.2, 0.8]) + with subcol1: + num = st.number_input("Max Images Displayed", min_value=0, max_value=len(st.session_state.IMGS), value=min(250, len(st.session_state.IMGS))) + query_form() + + if st.session_state.IMGS: + selected_imgs = image_select(f"Total samples: {len(st.session_state.IMGS)}", images=st.session_state.IMGS[0:num], indices=st.session_state.SELECTED_IMGS, use_container_width=False) #noqa - clicked = image_select("Samples", images=st.session_state.IMGS[0:num]) #noqa + with col2: + similarity_form(selected_imgs) + display_labels = st.checkbox("Display Labels", value=False) def launch(): diff --git a/yoloexplorer/frontend/pages/embedding.py b/yoloexplorer/frontend/pages/embedding.py new file mode 100644 index 0000000..e69de29 diff --git a/yoloexplorer/frontend/states.py b/yoloexplorer/frontend/states.py index ec37164..66277bb 100644 --- a/yoloexplorer/frontend/states.py +++ b/yoloexplorer/frontend/states.py @@ -1,8 +1,12 @@ import streamlit as st +INDEX_PAGE_QUERY_FORM_KEY = "index_page_query_form" +INDEX_PAGE_SIMILARITY_FORM_KEY = "index_page_similarity_form" + def init_states(): st.session_state.EXPLORER = None st.session_state.IMGS = [] + st.session_state.SELECTED_IMGS = 0 def update_state(state, value): st.session_state[state] = value \ No newline at end of file From b15060ba4f14d8db4003ad13668208a1b8c08903 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 17 Jul 2023 19:04:51 +0530 Subject: [PATCH 13/27] update --- yoloexplorer/frontend/layout.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 3e34521..7b02dee 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -65,7 +65,7 @@ def find_similar_imgs(imgs, limit=25): def layout(): st.set_page_config(layout='wide') - col1, col2= st.columns([0.75, 0.25], gap="small") + col1, col2 = st.columns([0.75, 0.25], gap="small") reset_to_init_state() with col1: @@ -79,8 +79,11 @@ def layout(): with col2: similarity_form(selected_imgs) - display_labels = st.checkbox("Display Labels", value=False) + display_labels = st.checkbox("Labels", value=False) #noqa + # TODO + st.write("Coming soon") + st.write("Merge Datasets") def launch(): cmd = ["streamlit", "run", __file__, "--server.maxMessageSize", "1024"] From 69ce4d9bc6c6721346d7dd268d1501b745c35410 Mon Sep 17 00:00:00 2001 From: Onuralp SEZER Date: Sun, 16 Jul 2023 18:05:11 +0300 Subject: [PATCH 14/27] =?UTF-8?q?feat:=20=E2=9C=A8=20introduce=20pyproject?= =?UTF-8?q?=20toml=20file=20and=20pre=20commit=20with=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Onuralp SEZER --- .github/workflows/main.yml | 10 +--- .pre-commit-config.yaml | 36 ++++++++++++ MANIFEST.in | 2 +- README.md | 7 +-- pyproject.toml | 60 ++++++++++++++++++++ requirements.txt | 2 +- setup.py | 3 +- tests/test_explorer.py | 20 ++----- yoloexplorer/__init__.py | 5 +- yoloexplorer/dataset.py | 4 +- yoloexplorer/explorer.py | 109 ++++++++++--------------------------- 11 files changed, 141 insertions(+), 117 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21e3a4d..4859d07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,16 +23,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ruff pytest + pip install pytest pip install git+https://github.com/ultralytics/ultralytics.git@embeddings pip install -e . - - - name: Lint with ruff - run: | - # stop the build if there are Python syntax errors or undefined names - ruff --format=github --select=E9,F63,F7,F82 --target-version=py37 --line-length=120 . - # default set of ruff rules with GitHub Annotations - ruff --format=github --target-version=py37 --line-length=120 . + - name: Test with pytest run: | pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7b3f55f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +default_language_version: + python: python3.8 +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-toml + - id: check-yaml + args: + - --unsafe + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/asottile/pyupgrade + rev: v3.7.0 + hooks: + - id: pyupgrade + args: + - --py3-plus + - --keep-runtime-typing +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.275 + hooks: + - id: ruff + args: + - --fix +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black +ci: + autofix_commit_msg: "fix(pre_commit): 🎨 auto format pre-commit hooks" + autoupdate_commit_msg: "fix(pre_commit): ⬆ pre_commit autoupdate" diff --git a/MANIFEST.in b/MANIFEST.in index 540b720..f9bd145 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt \ No newline at end of file +include requirements.txt diff --git a/README.md b/README.md index 6b4498d..f0b469e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # YOLOExplorer -Explore, manipulate and iterate on Computer Vision datasets with precision using simple APIs. +Explore, manipulate and iterate on Computer Vision datasets with precision using simple APIs. Supports SQL filters, vector similarity search, native interface with Pandas and more. @@ -111,12 +111,12 @@ coco_exp.remove_imgs([100,120,300..n]) # Removes images at the given ids. Adding data
For adding data from another dataset, you need an explorer object of that dataset with embeddings built. You can then pass that object along with the ids of the imgs that you'd like to add from that dataset. ``` -coco_exp.add_imgs(exp, idxs) # +coco_exp.add_imgs(exp, idxs) # ``` Note: You can use SQL querying and/or similarity searches to get the desired ids from the datasets. Persisting the Table: Create new dataset and start training
-After making the desired changes, you can persist the table to create the new dataset. +After making the desired changes, you can persist the table to create the new dataset. ``` coco_exp.persist() ``` @@ -175,4 +175,3 @@ Pre-filtering will enable powerful queries like - "Show me images similar to 10 else max(2,len(resized_images)) + cols = 10 if len(resized_images) > 10 else max(2, len(resized_images)) rows = max(1, math.ceil(len(resized_images) / cols)) fig, axes = plt.subplots(nrows=rows, ncols=cols) fig.subplots_adjust(hspace=0, wspace=0) @@ -267,11 +245,8 @@ def plot_imgs(self, ids=None, query=None, labels=True): ax.axis("off") # Display the grid of images plt.show() - - def get_similarity_index( - self, top_k=0.01, sim_thres=0.90, reduce=False, sorted=False - ): + def get_similarity_index(self, top_k=0.01, sim_thres=0.90, reduce=False, sorted=False): """ Args: @@ -283,9 +258,7 @@ def get_similarity_index( np.array: Similarity index """ if self.table is None: - LOGGER.error( - "No embedding space found. Please build the embedding space first." - ) + LOGGER.error("No embedding space found. Please build the embedding space first.") return None if top_k > 1.0: LOGGER.warning("top_k should be between 0 and 1. Setting top_k to 1.0") @@ -295,14 +268,10 @@ def get_similarity_index( top_k = 0.0 if sim_thres is not None: if sim_thres > 1.0: - LOGGER.warning( - "sim_thres should be between 0 and 1. Setting sim_thres to 1.0" - ) + LOGGER.warning("sim_thres should be between 0 and 1. Setting sim_thres to 1.0") sim_thres = 1.0 if sim_thres < 0.0: - LOGGER.warning( - "sim_thres should be between 0 and 1. Setting sim_thres to 0.0" - ) + LOGGER.warning("sim_thres should be between 0 and 1. Setting sim_thres to 0.0") sim_thres = 0.0 embs = np.array(self.table.to_arrow()["vector"].to_pylist()) self._sim_index = np.zeros(len(embs)) @@ -317,12 +286,8 @@ def get_similarity_index( dim = embs.shape[1] values = pa.array(embs.reshape(-1), type=pa.float32()) table_data = pa.FixedSizeListArray.from_arrays(values, dim) - table = pa.table( - [table_data, self.table.to_arrow()["id"]], names=["vector", "id"] - ) - self._search_table = self._create_table( - "reduced_embs", data=table, mode="overwrite" - ) + table = pa.table([table_data, self.table.to_arrow()["id"]], names=["vector", "id"]) + self._search_table = self._create_table("reduced_embs", data=table, mode="overwrite") # with multiprocessing.Pool() as pool: # multiprocessing doesn't do much. Need to revisit # list(tqdm(pool.imap(build_index, iterable))) @@ -337,9 +302,7 @@ def get_similarity_index( return self._sim_index if not sorted else np.sort(self._sim_index) - def plot_similarity_index( - self, sim_thres=0.90, top_k=0.01, reduce=False, sorted=False - ): + def plot_similarity_index(self, sim_thres=0.90, top_k=0.01, reduce=False, sorted=False): """ Plots the similarity index @@ -376,10 +339,8 @@ def remove_imgs(self, idxs): table = pa_table.filter(mask) ids = [i for i in range(len(table))] - table = table.set_column(0, 'id', [ids]) # TODO: Revisit this. This is a hack to fix the ids==dix - self.table = self._create_table( - self.temp_table_name, data=table, mode="overwrite" - ) # work on a temporary table + table = table.set_column(0, "id", [ids]) # TODO: Revisit this. This is a hack to fix the ids==dix + self.table = self._create_table(self.temp_table_name, data=table, mode="overwrite") # work on a temporary table self.log_status() @@ -392,9 +353,7 @@ def add_imgs(self, exp, idxs): """ table_df = self.table.to_pandas() data = exp.table.to_pandas().iloc[idxs] - assert len(table_df["vector"].iloc[0]) == len( - data["vector"].iloc[0] - ), "Vector dimension mismatch" + assert len(table_df["vector"].iloc[0]) == len(data["vector"].iloc[0]), "Vector dimension mismatch" table_df = pd.concat([table_df, data], ignore_index=True) ids = [i for i in range(len(table_df))] table_df["id"] = ids @@ -451,25 +410,17 @@ def persist(self, name=None): new_dataset_info = self.dataset_info.copy() new_dataset_info.pop("yaml_file") - new_dataset_info.pop( - "path" - ) # relative paths will get messed up when merging datasets - new_dataset_info.pop( - "download" - ) # Assume all files are present offline, there is no way to store metadata yet + new_dataset_info.pop("path") # relative paths will get messed up when merging datasets + new_dataset_info.pop("download") # Assume all files are present offline, there is no way to store metadata yet new_dataset_info["train"] = (path / train_txt).resolve().as_posix() for key, value in new_dataset_info.items(): if isinstance(value, Path): new_dataset_info[key] = value.as_posix() - yaml.dump( - new_dataset_info, open(path / datafile_name, "w") - ) # update dataset.yaml file + yaml.dump(new_dataset_info, open(path / datafile_name, "w")) # update dataset.yaml file # TODO: not sure if this should be called data_final to prevent overwriting the original data? - self.table = self._create_table( - self.table_name, data=self.table.to_arrow(), mode="overwrite" - ) + self.table = self._create_table(self.table_name, data=self.table.to_arrow(), mode="overwrite") db.drop_table(self.temp_table_name) LOGGER.info("Changes persisted to the dataset.") @@ -557,17 +508,13 @@ def _copy_table_to_project(self, table_path): name = Path(table_path).stem # lancedb doesn't need .lance extension db = lancedb.connect(path) table = db.open_table(name) - return self._create_table( - self.table_name, data=table.to_arrow(), mode="overwrite" - ) + return self._create_table(self.table_name, data=table.to_arrow(), mode="overwrite") def _embedding_func(self, imgs): embeddings = [] for img in tqdm(imgs): img = decode(img) - embeddings.append( - self.predictor.embed(img, verbose=self.verbose).squeeze().cpu().numpy() - ) + embeddings.append(self.predictor.embed(img, verbose=self.verbose).squeeze().cpu().numpy()) return embeddings def _setup_predictor(self, model, device=""): From 837d61ee4e6a0723148c8eed613a0998e1169208 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 03:18:41 +0530 Subject: [PATCH 15/27] udpate --- yoloexplorer/frontend/launch.py | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py new file mode 100644 index 0000000..a1f9ef8 --- /dev/null +++ b/yoloexplorer/frontend/launch.py @@ -0,0 +1,61 @@ +from collections import defaultdict as _defaultdict +from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator +from typing import Dict as _Dict +import streamlit as _st +import sys as _sys + + +# EXPORT +st = _st +ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) + + +# VARS +_IS_WRAPPED = False + + +def run_streamlit(python_file): + """ + IMPORTANT: import this function from a seperate file! + Use this function by placing a call to it at the top of your + check for __main__ + >>> from run_streamlit import run_streamlit, st, ST + >>> + >>> if __name__ == '__main__': + >>> run_streamlit(__file__) + >>> st.title('Hello') + >>> ST['info'].text('World!') + """ + + # do not wrap if streamlit is already running! + global _IS_WRAPPED + if _IS_WRAPPED: + return True + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + import click + import streamlit.cli + + @click.group() + def run_streamlit(): + pass + + # For some reason I cant get streamlit to work without this subcommand? + @run_streamlit.command('streamlit') + @streamlit.cli.configurator_options + def run_streamlit_subcommand(**kwargs): + global _IS_WRAPPED + _IS_WRAPPED = True + streamlit.cli._apply_config_options_from_cli(kwargs) + streamlit.cli._main_run(python_file, args) + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # swap out arguments and keep arguments, first is the filename + args = _sys.argv[1:] + _sys.argv = [_sys.argv[0], 'streamlit'] + + # run streamlit + run_streamlit() + exit(0) \ No newline at end of file From 589e1c5ac1c55410ebad2bfea36e05ecfc6278d7 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 23:59:18 +0530 Subject: [PATCH 16/27] update --- yoloexplorer/__init__.py | 4 +-- yoloexplorer/frontend/launch.py | 61 --------------------------------- yoloexplorer/frontend/layout.py | 3 -- yoloexplorer/frontend/states.py | 3 -- 4 files changed, 2 insertions(+), 69 deletions(-) delete mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index b775a9f..3f7404f 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,3 @@ from .explorer import Explorer - -__all__ = ["Explorer"] +from .frontend import layout +__all__ = ["Explorer", "layout"] diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py deleted file mode 100644 index a1f9ef8..0000000 --- a/yoloexplorer/frontend/launch.py +++ /dev/null @@ -1,61 +0,0 @@ -from collections import defaultdict as _defaultdict -from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator -from typing import Dict as _Dict -import streamlit as _st -import sys as _sys - - -# EXPORT -st = _st -ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) - - -# VARS -_IS_WRAPPED = False - - -def run_streamlit(python_file): - """ - IMPORTANT: import this function from a seperate file! - Use this function by placing a call to it at the top of your - check for __main__ - >>> from run_streamlit import run_streamlit, st, ST - >>> - >>> if __name__ == '__main__': - >>> run_streamlit(__file__) - >>> st.title('Hello') - >>> ST['info'].text('World!') - """ - - # do not wrap if streamlit is already running! - global _IS_WRAPPED - if _IS_WRAPPED: - return True - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - import click - import streamlit.cli - - @click.group() - def run_streamlit(): - pass - - # For some reason I cant get streamlit to work without this subcommand? - @run_streamlit.command('streamlit') - @streamlit.cli.configurator_options - def run_streamlit_subcommand(**kwargs): - global _IS_WRAPPED - _IS_WRAPPED = True - streamlit.cli._apply_config_options_from_cli(kwargs) - streamlit.cli._main_run(python_file, args) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # swap out arguments and keep arguments, first is the filename - args = _sys.argv[1:] - _sys.argv = [_sys.argv[0], 'streamlit'] - - # run streamlit - run_streamlit() - exit(0) \ No newline at end of file diff --git a/yoloexplorer/frontend/layout.py b/yoloexplorer/frontend/layout.py index 7b02dee..aa40d26 100644 --- a/yoloexplorer/frontend/layout.py +++ b/yoloexplorer/frontend/layout.py @@ -81,9 +81,6 @@ def layout(): similarity_form(selected_imgs) display_labels = st.checkbox("Labels", value=False) #noqa - # TODO - st.write("Coming soon") - st.write("Merge Datasets") def launch(): cmd = ["streamlit", "run", __file__, "--server.maxMessageSize", "1024"] diff --git a/yoloexplorer/frontend/states.py b/yoloexplorer/frontend/states.py index 66277bb..a3c8342 100644 --- a/yoloexplorer/frontend/states.py +++ b/yoloexplorer/frontend/states.py @@ -7,6 +7,3 @@ def init_states(): st.session_state.EXPLORER = None st.session_state.IMGS = [] st.session_state.SELECTED_IMGS = 0 - -def update_state(state, value): - st.session_state[state] = value \ No newline at end of file From f9e0f9d8cfad67c2798407642f851cc3ba4d311c Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 00:38:26 +0530 Subject: [PATCH 17/27] update --- yoloexplorer/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index 3f7404f..460bde0 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,4 @@ from .explorer import Explorer from .frontend import layout -__all__ = ["Explorer", "layout"] +from . import config +__all__ = ["Explorer", "layout", "config"] From 6777569c7aeee7f3ac536732fbbea1f3bfdb24ae Mon Sep 17 00:00:00 2001 From: Onuralp SEZER Date: Sun, 16 Jul 2023 18:05:11 +0300 Subject: [PATCH 18/27] =?UTF-8?q?feat:=20=E2=9C=A8=20introduce=20pyproject?= =?UTF-8?q?=20toml=20file=20and=20pre=20commit=20with=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Onuralp SEZER --- yoloexplorer/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index 460bde0..b775a9f 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,4 +1,3 @@ from .explorer import Explorer -from .frontend import layout -from . import config -__all__ = ["Explorer", "layout", "config"] + +__all__ = ["Explorer"] From 96db1781a5c0044e7ba276d1a2f030b74117845d Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 03:18:41 +0530 Subject: [PATCH 19/27] udpate --- yoloexplorer/frontend/launch.py | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py new file mode 100644 index 0000000..a1f9ef8 --- /dev/null +++ b/yoloexplorer/frontend/launch.py @@ -0,0 +1,61 @@ +from collections import defaultdict as _defaultdict +from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator +from typing import Dict as _Dict +import streamlit as _st +import sys as _sys + + +# EXPORT +st = _st +ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) + + +# VARS +_IS_WRAPPED = False + + +def run_streamlit(python_file): + """ + IMPORTANT: import this function from a seperate file! + Use this function by placing a call to it at the top of your + check for __main__ + >>> from run_streamlit import run_streamlit, st, ST + >>> + >>> if __name__ == '__main__': + >>> run_streamlit(__file__) + >>> st.title('Hello') + >>> ST['info'].text('World!') + """ + + # do not wrap if streamlit is already running! + global _IS_WRAPPED + if _IS_WRAPPED: + return True + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + import click + import streamlit.cli + + @click.group() + def run_streamlit(): + pass + + # For some reason I cant get streamlit to work without this subcommand? + @run_streamlit.command('streamlit') + @streamlit.cli.configurator_options + def run_streamlit_subcommand(**kwargs): + global _IS_WRAPPED + _IS_WRAPPED = True + streamlit.cli._apply_config_options_from_cli(kwargs) + streamlit.cli._main_run(python_file, args) + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # swap out arguments and keep arguments, first is the filename + args = _sys.argv[1:] + _sys.argv = [_sys.argv[0], 'streamlit'] + + # run streamlit + run_streamlit() + exit(0) \ No newline at end of file From 77f383844ea152a1fffeaf05d4d34c7d7cd36487 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 23:59:18 +0530 Subject: [PATCH 20/27] update --- yoloexplorer/__init__.py | 4 +-- yoloexplorer/frontend/launch.py | 61 --------------------------------- 2 files changed, 2 insertions(+), 63 deletions(-) delete mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index b775a9f..3f7404f 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,3 @@ from .explorer import Explorer - -__all__ = ["Explorer"] +from .frontend import layout +__all__ = ["Explorer", "layout"] diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py deleted file mode 100644 index a1f9ef8..0000000 --- a/yoloexplorer/frontend/launch.py +++ /dev/null @@ -1,61 +0,0 @@ -from collections import defaultdict as _defaultdict -from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator -from typing import Dict as _Dict -import streamlit as _st -import sys as _sys - - -# EXPORT -st = _st -ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) - - -# VARS -_IS_WRAPPED = False - - -def run_streamlit(python_file): - """ - IMPORTANT: import this function from a seperate file! - Use this function by placing a call to it at the top of your - check for __main__ - >>> from run_streamlit import run_streamlit, st, ST - >>> - >>> if __name__ == '__main__': - >>> run_streamlit(__file__) - >>> st.title('Hello') - >>> ST['info'].text('World!') - """ - - # do not wrap if streamlit is already running! - global _IS_WRAPPED - if _IS_WRAPPED: - return True - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - import click - import streamlit.cli - - @click.group() - def run_streamlit(): - pass - - # For some reason I cant get streamlit to work without this subcommand? - @run_streamlit.command('streamlit') - @streamlit.cli.configurator_options - def run_streamlit_subcommand(**kwargs): - global _IS_WRAPPED - _IS_WRAPPED = True - streamlit.cli._apply_config_options_from_cli(kwargs) - streamlit.cli._main_run(python_file, args) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # swap out arguments and keep arguments, first is the filename - args = _sys.argv[1:] - _sys.argv = [_sys.argv[0], 'streamlit'] - - # run streamlit - run_streamlit() - exit(0) \ No newline at end of file From 6a4910ba092991d40c16e0ade64bfb7dcf502300 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 00:38:26 +0530 Subject: [PATCH 21/27] update --- yoloexplorer/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index 3f7404f..460bde0 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,4 @@ from .explorer import Explorer from .frontend import layout -__all__ = ["Explorer", "layout"] +from . import config +__all__ = ["Explorer", "layout", "config"] From f8d2f3e685446bffef04a0bd6c7d48e3bd6d192c Mon Sep 17 00:00:00 2001 From: Onuralp SEZER Date: Sun, 16 Jul 2023 18:05:11 +0300 Subject: [PATCH 22/27] =?UTF-8?q?feat:=20=E2=9C=A8=20introduce=20pyproject?= =?UTF-8?q?=20toml=20file=20and=20pre=20commit=20with=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Onuralp SEZER --- yoloexplorer/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index 460bde0..b775a9f 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,4 +1,3 @@ from .explorer import Explorer -from .frontend import layout -from . import config -__all__ = ["Explorer", "layout", "config"] + +__all__ = ["Explorer"] From 74eae5cf5faa5b2b389a814b8816f8c973b23b75 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 03:18:41 +0530 Subject: [PATCH 23/27] udpate --- yoloexplorer/frontend/launch.py | 61 +++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py new file mode 100644 index 0000000..a1f9ef8 --- /dev/null +++ b/yoloexplorer/frontend/launch.py @@ -0,0 +1,61 @@ +from collections import defaultdict as _defaultdict +from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator +from typing import Dict as _Dict +import streamlit as _st +import sys as _sys + + +# EXPORT +st = _st +ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) + + +# VARS +_IS_WRAPPED = False + + +def run_streamlit(python_file): + """ + IMPORTANT: import this function from a seperate file! + Use this function by placing a call to it at the top of your + check for __main__ + >>> from run_streamlit import run_streamlit, st, ST + >>> + >>> if __name__ == '__main__': + >>> run_streamlit(__file__) + >>> st.title('Hello') + >>> ST['info'].text('World!') + """ + + # do not wrap if streamlit is already running! + global _IS_WRAPPED + if _IS_WRAPPED: + return True + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + import click + import streamlit.cli + + @click.group() + def run_streamlit(): + pass + + # For some reason I cant get streamlit to work without this subcommand? + @run_streamlit.command('streamlit') + @streamlit.cli.configurator_options + def run_streamlit_subcommand(**kwargs): + global _IS_WRAPPED + _IS_WRAPPED = True + streamlit.cli._apply_config_options_from_cli(kwargs) + streamlit.cli._main_run(python_file, args) + + # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # swap out arguments and keep arguments, first is the filename + args = _sys.argv[1:] + _sys.argv = [_sys.argv[0], 'streamlit'] + + # run streamlit + run_streamlit() + exit(0) \ No newline at end of file From 9f782ba518aedc5559c8ea6e3a192d3d3e8aca41 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Fri, 14 Jul 2023 23:59:18 +0530 Subject: [PATCH 24/27] update --- yoloexplorer/__init__.py | 4 +-- yoloexplorer/frontend/launch.py | 61 --------------------------------- 2 files changed, 2 insertions(+), 63 deletions(-) delete mode 100644 yoloexplorer/frontend/launch.py diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index b775a9f..3f7404f 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,3 @@ from .explorer import Explorer - -__all__ = ["Explorer"] +from .frontend import layout +__all__ = ["Explorer", "layout"] diff --git a/yoloexplorer/frontend/launch.py b/yoloexplorer/frontend/launch.py deleted file mode 100644 index a1f9ef8..0000000 --- a/yoloexplorer/frontend/launch.py +++ /dev/null @@ -1,61 +0,0 @@ -from collections import defaultdict as _defaultdict -from streamlit.delta_generator import DeltaGenerator as _DeltaGenerator -from typing import Dict as _Dict -import streamlit as _st -import sys as _sys - - -# EXPORT -st = _st -ST: _Dict[str, _DeltaGenerator] = _defaultdict(_st.empty) - - -# VARS -_IS_WRAPPED = False - - -def run_streamlit(python_file): - """ - IMPORTANT: import this function from a seperate file! - Use this function by placing a call to it at the top of your - check for __main__ - >>> from run_streamlit import run_streamlit, st, ST - >>> - >>> if __name__ == '__main__': - >>> run_streamlit(__file__) - >>> st.title('Hello') - >>> ST['info'].text('World!') - """ - - # do not wrap if streamlit is already running! - global _IS_WRAPPED - if _IS_WRAPPED: - return True - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - import click - import streamlit.cli - - @click.group() - def run_streamlit(): - pass - - # For some reason I cant get streamlit to work without this subcommand? - @run_streamlit.command('streamlit') - @streamlit.cli.configurator_options - def run_streamlit_subcommand(**kwargs): - global _IS_WRAPPED - _IS_WRAPPED = True - streamlit.cli._apply_config_options_from_cli(kwargs) - streamlit.cli._main_run(python_file, args) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # swap out arguments and keep arguments, first is the filename - args = _sys.argv[1:] - _sys.argv = [_sys.argv[0], 'streamlit'] - - # run streamlit - run_streamlit() - exit(0) \ No newline at end of file From 703e94fc0354a3b9b8da91f7658253f462cbc0f2 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Sat, 15 Jul 2023 00:38:26 +0530 Subject: [PATCH 25/27] update --- yoloexplorer/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yoloexplorer/__init__.py b/yoloexplorer/__init__.py index 3f7404f..460bde0 100644 --- a/yoloexplorer/__init__.py +++ b/yoloexplorer/__init__.py @@ -1,3 +1,4 @@ from .explorer import Explorer from .frontend import layout -__all__ = ["Explorer", "layout"] +from . import config +__all__ = ["Explorer", "layout", "config"] From 9d3481f90696192a95be82a005ef117b0977a84d Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 17 Jul 2023 19:41:15 +0530 Subject: [PATCH 26/27] update --- yoloexplorer/frontend/states.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yoloexplorer/frontend/states.py b/yoloexplorer/frontend/states.py index a3c8342..c745e9a 100644 --- a/yoloexplorer/frontend/states.py +++ b/yoloexplorer/frontend/states.py @@ -6,4 +6,7 @@ def init_states(): st.session_state.EXPLORER = None st.session_state.IMGS = [] - st.session_state.SELECTED_IMGS = 0 + st.session_state.SELECTED_IMGS = [] + +def update_state(state, value): + st.session_state[state] = value \ No newline at end of file From 3a0816d14a5634c06a2c1c63f42359fc09fa74ce Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 17 Jul 2023 19:42:57 +0530 Subject: [PATCH 27/27] update --- frontend/dashboard.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 frontend/dashboard.py diff --git a/frontend/dashboard.py b/frontend/dashboard.py new file mode 100644 index 0000000..d320398 --- /dev/null +++ b/frontend/dashboard.py @@ -0,0 +1,47 @@ +import streamlit as st +from yoloexplorer import Explorer +from streamlit_image_select import image_select +from typing import List + +st.title("YOLOExplorer") + +with st.sidebar: + default_data_yaml = "coco128.yaml" + st.text(f"Dataset: {default_data_yaml}") + if default_data_yaml: + if "data_yaml" not in st.session_state: + st.session_state["data_yaml"] = default_data_yaml + if "session" not in st.session_state: + st.session_state["session"] = Explorer(default_data_yaml, model=None) + dataset_info = st.session_state["session"].dataset_info + + st.write("**Dataset Information :**") + + for dataset_type in ["train", "val", "tets"]: + st.text(f"{dataset_type.capitalize()} paths :") + if isinstance(dataset_info["train"], str): + st.text(dataset_info["train"]) + elif isinstance(dataset_info["train", List]): + for p in dataset_info["train"]: + st.text(p) + +st.write("**Dataset Visualization**") + +# TODO: Only show one of the sample dataset at same time +img_train_checkbox_w, img_val_checkbox_w, img_test_checkbox_w, num_sample_w = st.columns(4) +with img_train_checkbox_w: + img_train_checkbox = st.checkbox(label="Train") +with img_val_checkbox_w: + img_val_checkbox = st.checkbox(label="Val") +with img_test_checkbox_w: + img_test_checkbox = st.checkbox(label="Test") +with num_sample_w: + num_samples_show = st.number_input(label="Numbers of Images", value=10) + + +if num_samples_show: + if img_train_checkbox: + img_url = "/home/hd/zidane.jpg" # Put path to your image + train_image_paths = [img_url] * int(num_samples_show) + # st.image(train_image_paths, use_column_width="always") + clicked = image_select("Training Samples", images=train_image_paths) \ No newline at end of file