From 74a1464a25cde088464b7391ac5178da772b0c39 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Mon, 25 May 2026 11:32:30 +0800 Subject: [PATCH 01/16] Add local Docker setup for NAS photos --- .gitignore | 3 +- docker-compose.local.yml | 76 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 docker-compose.local.yml diff --git a/.gitignore b/.gitignore index b6b1caa8..bb58c89c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,6 @@ dist .trae *uploads* data/* +pg_data/ .env -.claude/ \ No newline at end of file +.claude/ diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 00000000..949dca33 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,76 @@ +services: + postgres: + image: pgvector/pgvector:pg18-trixie + container_name: trailsnap_postgres + restart: unless-stopped + environment: + TZ: Asia/Shanghai + POSTGRES_DB: trailsnap + POSTGRES_USER: trailsnap + POSTGRES_PASSWORD: trailsnap + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + PGDATA: /var/lib/postgresql/data/pgdata + networks: + - app-network + ports: + - "5532:5432" + volumes: + - ./pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U trailsnap -d trailsnap -p 5432"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s + + server: + image: siyuan044/trailsnap-server:latest + restart: unless-stopped + expose: + - "8000" + ports: + - "8800:8000" + networks: + - app-network + volumes: + - ./data:/app/data + - /Volumes/photo:/app/Photos/:ro + environment: + - TZ=Asia/Shanghai + - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap + - RAILWAY_DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/railway + - AI_API_URL=http://ai:8001 + depends_on: + postgres: + condition: service_healthy + restart: true + + ai: + image: siyuan044/trailsnap-ai:latest + restart: unless-stopped + expose: + - "8001" + ports: + - "8801:8001" + networks: + - app-network + volumes: + - ./data:/app/data + environment: + - TZ=Asia/Shanghai + + frontend: + image: siyuan044/trailsnap-frontend:latest + restart: unless-stopped + ports: + - "8082:80" + depends_on: + - server + networks: + - app-network + environment: + - TZ=Asia/Shanghai + +networks: + app-network: + driver: bridge From cbed31c0f18aa8ef90570f2416f1acceede35ee8 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Mon, 25 May 2026 20:58:22 +0800 Subject: [PATCH 02/16] Stabilize local NAS task processing --- .gitignore | 1 + docker-compose.local.yml | 7 ++++++- package/server/app/service/task_worker.py | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index bb58c89c..22e5878c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ dist *uploads* data/* pg_data/ +backups/ .env .claude/ diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 949dca33..1c9262af 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -15,7 +15,7 @@ services: ports: - "5532:5432" volumes: - - ./pg_data:/var/lib/postgresql/data + - trailsnap_pg_data:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U trailsnap -d trailsnap -p 5432"] interval: 5s @@ -34,6 +34,7 @@ services: - app-network volumes: - ./data:/app/data + - ./package/server/app:/app/app:ro - /Volumes/photo:/app/Photos/:ro environment: - TZ=Asia/Shanghai @@ -74,3 +75,7 @@ services: networks: app-network: driver: bridge + +volumes: + trailsnap_pg_data: + external: true diff --git a/package/server/app/service/task_worker.py b/package/server/app/service/task_worker.py index 0eb352e2..b48dd6b8 100644 --- a/package/server/app/service/task_worker.py +++ b/package/server/app/service/task_worker.py @@ -60,6 +60,8 @@ def task_done(self, category: str): def get_chunk_size(task_type): chunk_size = 8 + if task_type == TaskType.PROCESS_BASIC: + chunk_size = 1 if task_type == TaskType.VISUAL_DESCRIPTION: chunk_size = 2 elif task_type == TaskType.OCR: @@ -289,10 +291,12 @@ def _fetch_tasks_to_queues_sync(self, allowed_types: List[str], current_qsizes: tasks_by_type = {} try: # We will fetch up to max_batch_size per category if its queue is below threshold - # Max items in queue per category - QUEUE_THRESHOLD = 50 - # How many items to fetch in one DB query per category - FETCH_BATCH_SIZE = 48 + # Keep the local Docker setup gentle. The original queue sizes can + # mark hundreds of tasks as PROCESSING before they actually run, + # which is painful when NAS IO or Docker Desktop storage stalls. + max_tasks = max(1, system_config.config.task.max_concurrent_tasks) + QUEUE_THRESHOLD = max_tasks + FETCH_BATCH_SIZE = max_tasks * 8 # Filter allowed types based on current queue size types_to_fetch = [] @@ -310,7 +314,7 @@ def _fetch_tasks_to_queues_sync(self, allowed_types: List[str], current_qsizes: query = query.filter(Task.type.in_(types_to_fetch)) # Fetch tasks. We fetch a bit more to fill the queues up - tasks = query.order_by(Task.priority.desc(), Task.created_at.asc()).limit(FETCH_BATCH_SIZE * 3).all() + tasks = query.order_by(Task.priority.desc(), Task.created_at.asc()).limit(FETCH_BATCH_SIZE).all() if tasks: for task in tasks: @@ -347,10 +351,11 @@ async def consumer_loop(self, category: str): # Configure max concurrency per consumer category based on system settings # or Fast Mode. Using Semaphores to allow multiple batches to run concurrently. max_concurrency = 1 + configured_workers = max(1, system_config.config.task.max_concurrent_tasks) if category == 'CPU': - max_concurrency = os.cpu_count() or 4 + max_concurrency = configured_workers elif category == 'IO': - max_concurrency = 10 + max_concurrency = configured_workers elif category == 'AI': max_concurrency = 1 @@ -650,4 +655,4 @@ def add_task(self, db: Session, type: str, payload: dict, priority: int = 0, own def add_tasks(self, db: Session, tasks_data: List[Dict], owner_id: UUID = None): """Batch add tasks""" - crud_task.add_tasks(db, tasks_data, owner_id) \ No newline at end of file + crud_task.add_tasks(db, tasks_data, owner_id) From ee0ef18cbfd055b613ab66e7e198163f5f307b10 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Mon, 25 May 2026 21:27:29 +0800 Subject: [PATCH 03/16] Enable Mimo visual description processing --- package/server/app/service/task_worker.py | 33 ++++++++++--------- .../app/service/tasks/visual_description.py | 23 ++++++++----- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/package/server/app/service/task_worker.py b/package/server/app/service/task_worker.py index b48dd6b8..1a636c17 100644 --- a/package/server/app/service/task_worker.py +++ b/package/server/app/service/task_worker.py @@ -296,37 +296,40 @@ def _fetch_tasks_to_queues_sync(self, allowed_types: List[str], current_qsizes: # which is painful when NAS IO or Docker Desktop storage stalls. max_tasks = max(1, system_config.config.task.max_concurrent_tasks) QUEUE_THRESHOLD = max_tasks - FETCH_BATCH_SIZE = max_tasks * 8 + FETCH_BATCH_SIZE = max_tasks - # Filter allowed types based on current queue size - types_to_fetch = [] + # Filter allowed types based on current queue size, then fetch per + # category. A single global priority query lets PROCESS_BASIC fill + # every fetch window and starve AI work such as VISUAL_DESCRIPTION. + types_by_category = {} for t in allowed_types: task_Factory = TaskStrategyFactory.get_strategy(t) if not task_Factory: continue cat = task_Factory.task_category if cat and current_qsizes.get(cat, 0) < QUEUE_THRESHOLD: - types_to_fetch.append(t) + types_by_category.setdefault(cat, []).append(t) - if not types_to_fetch: + if not types_by_category: return {} - query = db.query(Task).filter(Task.status == TaskStatus.PENDING) - query = query.filter(Task.type.in_(types_to_fetch)) - - # Fetch tasks. We fetch a bit more to fill the queues up - tasks = query.order_by(Task.priority.desc(), Task.created_at.asc()).limit(FETCH_BATCH_SIZE).all() - - if tasks: + for category, types_to_fetch in types_by_category.items(): + tasks = ( + db.query(Task) + .filter(Task.status == TaskStatus.PENDING) + .filter(Task.type.in_(types_to_fetch)) + .order_by(Task.priority.desc(), Task.created_at.asc()) + .limit(FETCH_BATCH_SIZE) + .all() + ) for task in tasks: - cat = TaskStrategyFactory.get_strategy(task.type).task_category - if not cat: continue - if task.type not in tasks_by_type: tasks_by_type[task.type] = [] task.status = TaskStatus.PROCESSING self.last_active_time[task.type] = datetime.now() tasks_by_type[task.type].append({'id': task.id, 'type': task.type, 'priority': task.priority}) + + if tasks_by_type: db.commit() # Split tasks into smaller batches of max 8 items diff --git a/package/server/app/service/tasks/visual_description.py b/package/server/app/service/tasks/visual_description.py index 08d2c878..40ea0e22 100644 --- a/package/server/app/service/tasks/visual_description.py +++ b/package/server/app/service/tasks/visual_description.py @@ -51,17 +51,22 @@ def create_client(self, settings): logger.error("Visual Model connection has no api_key: %s", settings.analysis_connection_id) raise ValueError(f"Visual Model connection has no api_key: {settings.analysis_connection_id}") - # 2. Call OpenAI API - client = ChatOpenAI( - api_key=connection.api_key, - model= settings.analysis_model_name, - base_url=connection.api_base if connection.api_base else None, - timeout=60, - extra_body={ + # 2. Call OpenAI-compatible API. Some providers reject + # reasoning_effort="none"; disabling thinking through extra_body is + # enough for Xiaomi Mimo while still working with LM Studio Qwen. + client_kwargs = { + "api_key": connection.api_key, + "model": settings.analysis_model_name, + "base_url": connection.api_base if connection.api_base else None, + "timeout": 60, + "extra_body": { "chat_template_kwargs": {"enable_thinking": False}, }, - reasoning_effort="none", - ) + } + if "xiaomimimo.com" not in (connection.api_base or ""): + client_kwargs["reasoning_effort"] = "none" + + client = ChatOpenAI(**client_kwargs) return client async def process(self, worker, task: Task, db: Session) -> Dict[str, Any]: From 42c5fe8e2467e9d65d61bc9487fbf001fdf2476b Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Mon, 25 May 2026 21:31:54 +0800 Subject: [PATCH 04/16] Increase local task concurrency cautiously --- package/server/app/service/task_worker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package/server/app/service/task_worker.py b/package/server/app/service/task_worker.py index 1a636c17..77f42860 100644 --- a/package/server/app/service/task_worker.py +++ b/package/server/app/service/task_worker.py @@ -263,7 +263,8 @@ def _manage_pool_lifecycle(self): if active_count > 0: if active_cpu_count > 0 and self.process_pool is None: logging.info(f"Restarting process pool") - self.process_pool = concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) + max_workers = max(1, system_config.config.task.max_concurrent_tasks) + self.process_pool = concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) if self.thread_pool is None and active_io_count > 0: max_workers = system_config.config.task.max_concurrent_tasks logging.info(f"Restarting thread pool") @@ -360,7 +361,7 @@ async def consumer_loop(self, category: str): elif category == 'IO': max_concurrency = configured_workers elif category == 'AI': - max_concurrency = 1 + max_concurrency = configured_workers semaphore = asyncio.Semaphore(max_concurrency) From 5bea199729265644c962fe7994bee3eb2c013628 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Tue, 26 May 2026 02:34:32 +0800 Subject: [PATCH 05/16] Harden visual description JSON parsing --- .../app/service/tasks/visual_description.py | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/package/server/app/service/tasks/visual_description.py b/package/server/app/service/tasks/visual_description.py index 40ea0e22..21fe10fd 100644 --- a/package/server/app/service/tasks/visual_description.py +++ b/package/server/app/service/tasks/visual_description.py @@ -5,6 +5,7 @@ import os import json import base64 +import re from typing import Dict, Any, List from sqlalchemy.orm import Session from langchain_openai import ChatOpenAI @@ -19,6 +20,34 @@ logger = logging.getLogger(__name__) +def parse_json_response(content: str) -> Dict[str, Any]: + text = (content or "").strip() + if text.startswith("```"): + match = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL | re.IGNORECASE) + if match: + text = match.group(1).strip() + + candidates = [text] + brace_index = text.find("{") + if brace_index > 0: + candidates.append(text[brace_index:]) + + last_error = None + decoder = json.JSONDecoder(strict=False) + for candidate in candidates: + if not candidate: + continue + try: + result, _ = decoder.raw_decode(candidate.strip()) + if isinstance(result, dict): + return result + except json.JSONDecodeError as e: + last_error = e + + preview = text[:300].replace("\n", "\\n") + raise ValueError(f"Failed to parse model JSON response: {preview}") from last_error + + def encode_image(image_path): with open(image_path, "rb") as image_file: return base64.b64encode(image_file.read()).decode('utf-8') @@ -201,16 +230,10 @@ async def process_single_photo(self, worker, photo: Photo, db: Session, settings ] ) - eval_content = eval_response.content.strip().strip('`').strip().strip('json') - print(eval_content) - # Clean up code blocks if present - if eval_content.startswith("```"): - eval_content = eval_content.strip("`") - if eval_content.startswith("json"): - eval_content = eval_content[4:] + eval_content = eval_response.content try: - result_json = json.loads(eval_content.strip()) - except json.JSONDecodeError as e: + result_json = parse_json_response(eval_content) + except (json.JSONDecodeError, ValueError) as e: logger.error(f"Failed to parse evaluation JSON for photo {photo.id}: {eval_content}") raise e From c23788ac34f55d0c3b806322c30076015edd7af7 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Tue, 26 May 2026 03:03:38 +0800 Subject: [PATCH 06/16] Skip high-risk visual description rejects --- .../app/service/tasks/visual_description.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/package/server/app/service/tasks/visual_description.py b/package/server/app/service/tasks/visual_description.py index 21fe10fd..d58243d6 100644 --- a/package/server/app/service/tasks/visual_description.py +++ b/package/server/app/service/tasks/visual_description.py @@ -53,6 +53,13 @@ def encode_image(image_path): return base64.b64encode(image_file.read()).decode('utf-8') +def mark_visual_description_processed(photo: Photo, db: Session) -> None: + tasks_status = dict(photo.processed_tasks or {}) + tasks_status['visual_description'] = True + photo.processed_tasks = tasks_status + db.commit() + + @TaskStrategyFactory.register(TaskType.VISUAL_DESCRIPTION) class VisualDescriptionStrategy(BaseTaskStrategy): @property @@ -231,6 +238,10 @@ async def process_single_photo(self, worker, photo: Photo, db: Session, settings ) eval_content = eval_response.content + if "considered high risk" in (eval_content or "").lower(): + mark_visual_description_processed(photo, db) + return {'status': 'skipped', 'reason': 'model rejected high risk image'} + try: result_json = parse_json_response(eval_content) except (json.JSONDecodeError, ValueError) as e: @@ -256,10 +267,7 @@ async def process_single_photo(self, worker, photo: Photo, db: Session, settings ) db.add(desc) # Update photo processed status - tasks_status = dict(photo.processed_tasks or {}) - tasks_status['visual_description'] = True - photo.processed_tasks = tasks_status - db.commit() + mark_visual_description_processed(photo, db) return { 'status': 'completed', 'description': desc.description, From 2962a98892594b14c31f7233edbfeeacf712e60c Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Tue, 26 May 2026 04:03:54 +0800 Subject: [PATCH 07/16] Recover partial visual description JSON --- .../app/service/tasks/visual_description.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/package/server/app/service/tasks/visual_description.py b/package/server/app/service/tasks/visual_description.py index d58243d6..17db6c98 100644 --- a/package/server/app/service/tasks/visual_description.py +++ b/package/server/app/service/tasks/visual_description.py @@ -20,6 +20,59 @@ logger = logging.getLogger(__name__) +def _extract_json_string_field(text: str, key: str) -> str | None: + match = re.search(rf'"{key}"\s*:\s*"', text) + if not match: + return None + + start = match.end() + next_key = re.search( + r',?\s*\n?\s*"(description|tags|memory_score|beauty_score|quality_score|reason|narrative)"\s*:', + text[start:], + ) + end = start + next_key.start() if next_key else len(text) + value = text[start:end].strip() + value = re.sub(r'"\s*,?\s*\}?\s*$', '', value, flags=re.DOTALL).strip() + return value.replace('\\"', '"') + + +def _extract_json_array_field(text: str, key: str) -> List[str] | None: + match = re.search(rf'"{key}"\s*:\s*(\[[^\]]*\])', text, re.DOTALL) + if not match: + return None + raw_value = match.group(1) + try: + value = json.loads(raw_value, strict=False) + if isinstance(value, list): + return value + except json.JSONDecodeError: + pass + return re.findall(r'"([^"]+)"', raw_value) + + +def _extract_json_number_field(text: str, key: str) -> float | None: + match = re.search(rf'"{key}"\s*:\s*(-?\d+(?:\.\d+)?)', text) + if not match: + return None + return float(match.group(1)) + + +def _parse_partial_json_response(text: str) -> Dict[str, Any]: + result = { + "description": _extract_json_string_field(text, "description"), + "tags": _extract_json_array_field(text, "tags") or [], + "memory_score": _extract_json_number_field(text, "memory_score"), + "beauty_score": _extract_json_number_field(text, "beauty_score"), + "quality_score": _extract_json_number_field(text, "quality_score"), + "reason": _extract_json_string_field(text, "reason"), + "narrative": _extract_json_string_field(text, "narrative"), + } + result = {key: value for key, value in result.items() if value not in (None, "")} + if "description" in result or "narrative" in result: + return result + raise ValueError("No usable fields found in partial JSON response") + + def parse_json_response(content: str) -> Dict[str, Any]: text = (content or "").strip() if text.startswith("```"): @@ -44,6 +97,14 @@ def parse_json_response(content: str) -> Dict[str, Any]: except json.JSONDecodeError as e: last_error = e + for candidate in candidates: + if not candidate: + continue + try: + return _parse_partial_json_response(candidate.strip()) + except ValueError: + pass + preview = text[:300].replace("\n", "\\n") raise ValueError(f"Failed to parse model JSON response: {preview}") from last_error From 521c6810f91d1508e9f2ea955fc9de0157ce5550 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Tue, 26 May 2026 12:22:35 +0800 Subject: [PATCH 08/16] Add NAS Docker Compose setup --- docker-compose.nas.yml | 80 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docker-compose.nas.yml diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml new file mode 100644 index 00000000..dda95237 --- /dev/null +++ b/docker-compose.nas.yml @@ -0,0 +1,80 @@ +services: + postgres: + image: pgvector/pgvector:pg18-trixie + container_name: trailsnap_postgres + restart: unless-stopped + environment: + TZ: Asia/Shanghai + POSTGRES_DB: trailsnap + POSTGRES_USER: trailsnap + POSTGRES_PASSWORD: trailsnap + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + PGDATA: /var/lib/postgresql/data/pgdata + networks: + - app-network + ports: + - "5532:5432" + volumes: + - trailsnap_pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U trailsnap -d trailsnap -p 5432"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s + + server: + image: siyuan044/trailsnap-server:latest + restart: unless-stopped + expose: + - "8000" + ports: + - "8800:8000" + networks: + - app-network + volumes: + - ./data:/app/data + - ./package/server/app:/app/app:ro + - /volume1/photo:/app/Photos/:ro + environment: + - TZ=Asia/Shanghai + - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap + - RAILWAY_DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/railway + - AI_API_URL=http://ai:8001 + depends_on: + postgres: + condition: service_healthy + restart: true + + ai: + image: siyuan044/trailsnap-ai:latest + restart: unless-stopped + expose: + - "8001" + ports: + - "8801:8001" + networks: + - app-network + volumes: + - ./data:/app/data + environment: + - TZ=Asia/Shanghai + + frontend: + image: siyuan044/trailsnap-frontend:latest + restart: unless-stopped + ports: + - "8082:80" + depends_on: + - server + networks: + - app-network + environment: + - TZ=Asia/Shanghai + +networks: + app-network: + driver: bridge + +volumes: + trailsnap_pg_data: From 7aa755cb62790bcab5cecacb06a6f44af6ec0ade Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Tue, 26 May 2026 14:13:06 +0800 Subject: [PATCH 09/16] Point NAS server to Mac AI service --- docker-compose.nas.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml index dda95237..1ff903f9 100644 --- a/docker-compose.nas.yml +++ b/docker-compose.nas.yml @@ -40,7 +40,7 @@ services: - TZ=Asia/Shanghai - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap - RAILWAY_DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/railway - - AI_API_URL=http://ai:8001 + - AI_API_URL=http://192.168.10.12:8801 depends_on: postgres: condition: service_healthy From 06df1f7b928a9c6111b9b6b4befb25b6cd1f7254 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Wed, 27 May 2026 20:59:01 +0800 Subject: [PATCH 10/16] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20TrailSnap=20?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E6=8E=A5=E5=8F=A3=20404?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.nas.yml | 3 + package/server/app/api/classification.py | 54 +++++++++++++ package/server/app/crud/tag.py | 75 ++++++++++++++++++- .../src/components/home/FaceSection.vue | 2 +- package/website/src/router/index.ts | 3 + 5 files changed, 135 insertions(+), 2 deletions(-) diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml index 1ff903f9..6d98c93d 100644 --- a/docker-compose.nas.yml +++ b/docker-compose.nas.yml @@ -34,6 +34,7 @@ services: - app-network volumes: - ./data:/app/data + - ./package/server/main.py:/app/main.py:ro - ./package/server/app:/app/app:ro - /volume1/photo:/app/Photos/:ro environment: @@ -65,6 +66,8 @@ services: restart: unless-stopped ports: - "8082:80" + volumes: + - ./package/website/dist:/usr/share/nginx/html:ro depends_on: - server networks: diff --git a/package/server/app/api/classification.py b/package/server/app/api/classification.py index fc807867..d30f4f78 100644 --- a/package/server/app/api/classification.py +++ b/package/server/app/api/classification.py @@ -16,6 +16,13 @@ class RemovePhotosRequest(BaseModel): photo_ids: List[UUID] +class RenameTagRequest(BaseModel): + new_name: str + +class MergeTagsRequest(BaseModel): + target_name: str + source_names: List[str] + @router.get("", response_model=BaseResponse[List[schemas.TagStats]], summary="获取智能分类标签列表") def get_tags( skip: int = 0, @@ -29,6 +36,53 @@ def get_tags( data = crud.get_tags_with_stats(db, current_user.id, skip, limit) return BaseResponse(data=data) +@router.post("/merge", summary="合并分类标签") +def merge_tags( + payload: MergeTagsRequest = Body(..., description="合并标签请求"), + db: Session = Depends(get_db), + current_user: User = Depends(deps.get_current_user), +): + success, count, message = crud.merge_tags( + db, + current_user.id, + payload.target_name, + payload.source_names, + ) + if not success: + raise HTTPException(status_code=400, detail=message) + + from app.crud.album import trigger_conditional_albums_update + trigger_conditional_albums_update(db, current_user.id, None) + return BaseResponse(data={"status": "success", "count": count}) + +@router.put("/{path:path}", summary="重命名分类标签") +def rename_tag( + payload: RenameTagRequest = Body(..., description="标签重命名请求"), + path: str = Path(..., description="标签名称（支持多级/包含/）", path=True), + db: Session = Depends(get_db), + current_user: User = Depends(deps.get_current_user), +): + success, message = crud.rename_tag(db, current_user.id, path, payload.new_name) + if not success: + raise HTTPException(status_code=400, detail=message) + + from app.crud.album import trigger_conditional_albums_update + trigger_conditional_albums_update(db, current_user.id, None) + return BaseResponse(data={"status": "success"}) + +@router.delete("/{path:path}", summary="删除分类标签") +def delete_tag( + path: str = Path(..., description="标签名称（支持多级/包含/）", path=True), + db: Session = Depends(get_db), + current_user: User = Depends(deps.get_current_user), +): + if not crud.delete_tag(db, current_user.id, path): + raise HTTPException(status_code=404, detail="Tag not found") + + from app.crud.album import trigger_conditional_albums_update + trigger_conditional_albums_update(db, current_user.id, None) + return BaseResponse(data={"status": "success"}) + @router.get("/{path:path}/photos", response_model=BaseResponse[List[photo_schemas.Photo]], summary="获取分类照片列表") def get_tag_photos( # path=True 声明：匹配剩余的全部路径（支持包含/） diff --git a/package/server/app/crud/tag.py b/package/server/app/crud/tag.py index 7abfd59c..7882645f 100644 --- a/package/server/app/crud/tag.py +++ b/package/server/app/crud/tag.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List, Optional, Tuple from uuid import UUID from sqlalchemy.orm import Session from app.db.models.tag import PhotoTag, PhotoTagRelation @@ -184,3 +184,76 @@ def set_tag_cover(db: Session, owner_id: UUID, tag_name: str, photo_id: UUID) -> tag.cover_id = photo_id db.commit() return True + +def delete_tag(db: Session, owner_id: UUID, tag_name: str) -> bool: + tag = get_tag_by_name(db, tag_name, owner_id) + if not tag: + return False + + db.query(PhotoTagRelation).filter(PhotoTagRelation.tag_id == tag.id).delete(synchronize_session=False) + db.delete(tag) + db.commit() + return True + +def rename_tag(db: Session, owner_id: UUID, old_name: str, new_name: str) -> Tuple[bool, str]: + new_name = new_name.strip() + if not new_name: + return False, "New tag name is empty" + + tag = get_tag_by_name(db, old_name, owner_id) + if not tag: + return False, "Tag not found" + + existing = get_tag_by_name(db, new_name, owner_id) + if existing and existing.id != tag.id: + return False, "Target tag already exists" + + tag.tag_name = new_name + db.commit() + return True, "success" + +def merge_tags(db: Session, owner_id: UUID, target_name: str, source_names: List[str]) -> Tuple[bool, int, str]: + target_name = target_name.strip() + source_names = [name.strip() for name in source_names if name and name.strip()] + if not target_name: + return False, 0, "Target tag name is empty" + if not source_names: + return False, 0, "Source tags are empty" + + target = get_tag_by_name(db, target_name, owner_id) + if not target: + target = create_tag(db, target_name, "classification", owner_id) + + merged_count = 0 + for source_name in source_names: + if source_name == target_name: + continue + + source = get_tag_by_name(db, source_name, owner_id) + if not source: + continue + + if not target.cover_id and source.cover_id: + target.cover_id = source.cover_id + + source_relations = db.query(PhotoTagRelation).filter( + PhotoTagRelation.tag_id == source.id + ).all() + + for relation in source_relations: + target_relation = db.query(PhotoTagRelation).filter( + PhotoTagRelation.photo_id == relation.photo_id, + PhotoTagRelation.tag_id == target.id + ).first() + + if target_relation: + target_relation.confidence = max(target_relation.confidence or 0, relation.confidence or 0) + db.delete(relation) + else: + relation.tag_id = target.id + merged_count += 1 + + db.delete(source) + + db.commit() + return True, merged_count, "success" diff --git a/package/website/src/components/home/FaceSection.vue b/package/website/src/components/home/FaceSection.vue index 9ffa2807..30ffdbcc 100644 --- a/package/website/src/components/home/FaceSection.vue +++ b/package/website/src/components/home/FaceSection.vue @@ -5,7 +5,7 @@

人物相册

{{ data.total_identified }}位已识别 diff --git a/package/website/src/router/index.ts b/package/website/src/router/index.ts index 1ff5ce35..76d94870 100644 --- a/package/website/src/router/index.ts +++ b/package/website/src/router/index.ts @@ -60,6 +60,9 @@ const routes: RouteRecordRaw[] = [ ], }, + { path: '/people', redirect: '/album/people' }, + { path: '/people/:id', redirect: to => `/album/people/${to.params.id}` }, + // Annual Report (Standalone Layout) { path: '/annual-report', From fc417e20e24e4801218046e0a74ad8613ceeca91 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Thu, 28 May 2026 08:41:04 +0800 Subject: [PATCH 11/16] =?UTF-8?q?fix:=20=E7=A8=B3=E5=AE=9A=E6=B8=85?= =?UTF-8?q?=E7=90=86=E9=A1=B5=E5=88=86=E9=A1=B5=E5=B9=B6=E5=88=A0=E9=99=A4?= =?UTF-8?q?=20RAW=20=E4=BC=B4=E7=94=9F=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.nas.yml | 2 +- package/server/app/api/toolbox.py | 34 +++++++--- package/server/app/service/storage.py | 67 +++++++++++++++++++ .../website/src/views/toolbox/CleanupPage.vue | 18 ++++- 4 files changed, 108 insertions(+), 13 deletions(-) diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml index 6d98c93d..d4dc3154 100644 --- a/docker-compose.nas.yml +++ b/docker-compose.nas.yml @@ -36,7 +36,7 @@ services: - ./data:/app/data - ./package/server/main.py:/app/main.py:ro - ./package/server/app:/app/app:ro - - /volume1/photo:/app/Photos/:ro + - /volume1/photo:/app/Photos/ environment: - TZ=Asia/Shanghai - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap diff --git a/package/server/app/api/toolbox.py b/package/server/app/api/toolbox.py index a8494dd8..353b643a 100644 --- a/package/server/app/api/toolbox.py +++ b/package/server/app/api/toolbox.py @@ -335,18 +335,34 @@ def get_photos_for_cleanup( db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): - # Join with ImageDescription to access scores - query = db.query(Photo).join(ImageDescriptionModel, Photo.id == ImageDescriptionModel.photo_id).filter(Photo.owner_id == current_user.id, Photo.is_deleted == False) - - # Calculate score: memory_score + quality_score - # We use coalesce to treat nulls as 0 - score_expr = func.coalesce(ImageDescriptionModel.memory_score, 0) + func.coalesce(ImageDescriptionModel.quality_score, 0) + score_subquery = db.query( + ImageDescriptionModel.photo_id.label("photo_id"), + func.max( + func.coalesce(ImageDescriptionModel.memory_score, 0) + + func.coalesce(ImageDescriptionModel.quality_score, 0) + ).label("score") + ).group_by(ImageDescriptionModel.photo_id).subquery() + + query = db.query(Photo).join( + score_subquery, + Photo.id == score_subquery.c.photo_id + ).filter( + Photo.owner_id == current_user.id, + Photo.is_deleted == False + ) if sort_by == "desc": - query = query.order_by(score_expr.desc()) + query = query.order_by( + score_subquery.c.score.desc(), + Photo.photo_time.desc().nulls_last(), + Photo.id.desc(), + ) else: - query = query.order_by(score_expr.asc()) + query = query.order_by( + score_subquery.c.score.asc(), + Photo.photo_time.asc().nulls_last(), + Photo.id.asc(), + ) photos = query.offset(skip).limit(limit).all() return BaseResponse(data=photos) - diff --git a/package/server/app/service/storage.py b/package/server/app/service/storage.py index 0ee067a1..f5b47acd 100644 --- a/package/server/app/service/storage.py +++ b/package/server/app/service/storage.py @@ -22,6 +22,11 @@ # Global cache for storage root (User ID -> Root Path) _STORAGE_ROOT_CACHE = {} +RAW_EXTENSIONS = { + '.arw', '.cr2', '.cr3', '.dng', '.nef', '.nrw', '.orf', + '.pef', '.raf', '.raw', '.rw2', '.srw', '.x3f' +} + def _get_storage_root(user_id: UUID, db: Session = None) -> str: try: root = './data/uploads' # Default path @@ -237,6 +242,67 @@ def get_live_photo_vide(image_path: str) -> Optional[str]: logging.error(f"Error getting live photo video for {image_path}: {e}") return None +def _raw_candidate_dirs(file_path: str): + parent = os.path.abspath(os.path.dirname(file_path)) + seen = set() + + def add(path: str): + path = os.path.abspath(path) + if path not in seen and os.path.isdir(path): + seen.add(path) + yield path + + yield from add(parent) + + def add_raw_dir_with_children(path: str): + if 'raw' not in os.path.basename(path).lower(): + return + yield from add(path) + try: + for name in os.listdir(path): + child = os.path.join(path, name) + if os.path.isdir(child): + yield from add(child) + except OSError as e: + logging.warning(f"Could not inspect RAW sidecar directory {path}: {e}") + + try: + for name in os.listdir(parent): + yield from add_raw_dir_with_children(os.path.join(parent, name)) + except OSError as e: + logging.warning(f"Could not inspect photo directory {parent}: {e}") + + grandparent = os.path.dirname(parent) + if grandparent and grandparent != parent: + try: + for name in os.listdir(grandparent): + yield from add_raw_dir_with_children(os.path.join(grandparent, name)) + except OSError as e: + logging.warning(f"Could not inspect sibling directories for {parent}: {e}") + +def delete_raw_sidecars(file_path: str): + base_name, ext = os.path.splitext(os.path.basename(file_path)) + if ext.lower() in RAW_EXTENSIONS: + return + + base_name_lower = base_name.lower() + for directory in _raw_candidate_dirs(file_path): + try: + names = os.listdir(directory) + except OSError as e: + logging.warning(f"Could not inspect RAW sidecar candidates in {directory}: {e}") + continue + + for name in names: + candidate_base, candidate_ext = os.path.splitext(name) + if candidate_base.lower() == base_name_lower and candidate_ext.lower() in RAW_EXTENSIONS: + candidate = os.path.join(directory, name) + try: + os.remove(candidate) + logging.info(f"Deleted RAW sidecar: {candidate}") + except OSError as e: + logging.error(f"Error deleting RAW sidecar {candidate}: {e}") + def delete_file(user_id: UUID, file_path: str, file_id: UUID, is_live_photo: bool = False): try: if os.path.exists(file_path): @@ -245,6 +311,7 @@ def delete_file(user_id: UUID, file_path: str, file_id: UUID, is_live_photo: boo video_path = get_live_photo_vide(file_path) if video_path and os.path.exists(video_path): os.remove(video_path) + delete_raw_sidecars(file_path) delete_thumbnails(user_id, file_id) except Exception as e: logging.error(f"Error deleting file {user_id}/{file_path}: {e}") diff --git a/package/website/src/views/toolbox/CleanupPage.vue b/package/website/src/views/toolbox/CleanupPage.vue index cbc05b8d..a4623158 100644 --- a/package/website/src/views/toolbox/CleanupPage.vue +++ b/package/website/src/views/toolbox/CleanupPage.vue @@ -114,6 +114,15 @@ const subtitle = computed(() => { return `${photos.value.length}${hasMore.value ? '+' : ''} 张照片` }) +const dedupePhotos = (items: AlbumImage[], existingIds = new Set()) => { + const seen = new Set(existingIds) + return items.filter(item => { + if (seen.has(item.id)) return false + seen.add(item.id) + return true + }) +} + const fetchPhotos = async (isLoadMore = false) => { if (loading.value) return @@ -132,18 +141,21 @@ const fetchPhotos = async (isLoadMore = false) => { }) const newPhotos = res.map(item => mapPhotoToImage(item)) + const uniqueNewPhotos = isLoadMore + ? dedupePhotos(newPhotos, new Set(photos.value.map(item => item.id))) + : dedupePhotos(newPhotos) if (newPhotos.length < limit) { hasMore.value = false } if (isLoadMore) { - photos.value.push(...newPhotos) + photos.value.push(...uniqueNewPhotos) } else { - photos.value = newPhotos + photos.value = uniqueNewPhotos } - skip.value += limit + skip.value += newPhotos.length } catch (e) { console.error('Fetch failed:', e) ElMessage.error('获取照片失败') From 0d854d6d6677fdc7b3c96a192efdbe9c45dae6cd Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Thu, 28 May 2026 08:58:31 +0800 Subject: [PATCH 12/16] fix: improve photo time recovery --- package/server/app/utils/exif.py | 190 ++++++++++++++++-- package/server/app/utils/filename.py | 8 +- .../tests/test_photo_time_extraction.py | 33 +++ 3 files changed, 214 insertions(+), 17 deletions(-) create mode 100644 package/server/tests/test_photo_time_extraction.py diff --git a/package/server/app/utils/exif.py b/package/server/app/utils/exif.py index 2bb0b6c3..224b7bde 100644 --- a/package/server/app/utils/exif.py +++ b/package/server/app/utils/exif.py @@ -9,6 +9,7 @@ @Description : """ import shutil +import subprocess import traceback from datetime import datetime import re @@ -26,6 +27,9 @@ from app.utils.filename import extract_datetime_from_filename +IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.tiff', '.webp', '.png', '.heic', '.heif') +VIDEO_EXTENSIONS = ('.mp4', '.mov', '.avi', '.mkv', '.webm', '.m4v') + # Helper Functions for Metadata # resources/rg_data RG_DIR = os.path.join(os.path.dirname(__file__), '../../resources/rg_data') @@ -168,16 +172,148 @@ def get_file_time_form_system(file_path: str) -> datetime: return datetime.now() +def parse_datetime_value(value: Any) -> Optional[datetime]: + if not value: + return None + + if isinstance(value, (bytes, bytearray)): + value = value.decode(errors="ignore") + + value = str(value).strip().replace("\x00", "") + formats = ( + "%Y:%m:%d %H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y:%m:%d %H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S%z", + ) + for fmt in formats: + try: + dt = datetime.strptime(value, fmt) + if dt.tzinfo: + dt = dt.astimezone().replace(tzinfo=None) + return dt + except ValueError: + continue + return None + + +def extract_datetime_from_exif(exif_data: Dict[str, Any]) -> tuple[Optional[datetime], Optional[str]]: + for tag_name in ("DateTimeOriginal", "DateTimeDigitized", "DateTime"): + dt = parse_datetime_value(exif_data.get(tag_name)) + if dt: + return dt, f"exif:{tag_name}" + return None, None + + +def _date_from_short_token(text: str) -> Optional[datetime]: + match = re.search(r"(? Optional[datetime]: + if not year: + return None + match = re.search(r"(? Optional[datetime]: + """ + Recover a coarse capture date from dated folders, e.g. 20250227, + 2024-3-28, 211004..., or 2020.09.13/.../9.15. + """ + inherited_year = None + best_dt = None + directory = os.path.dirname(file_path) + for part in [p for p in directory.split(os.sep) if p]: + dt = extract_datetime_from_filename(part) or _date_from_short_token(part) + if dt: + inherited_year = dt.year + best_dt = dt + continue + + month_day_dt = _date_from_month_day(part, inherited_year) + if month_day_dt: + best_dt = month_day_dt + + return best_dt + + +def extract_datetime_from_video_metadata(file_path: str) -> Optional[datetime]: + if not shutil.which("ffprobe"): + return None + + try: + result = subprocess.run( + [ + "ffprobe", + "-v", + "quiet", + "-print_format", + "json", + "-show_entries", + "format_tags=creation_time:stream_tags=creation_time", + file_path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=12, + ) + if result.returncode != 0 or not result.stdout.strip(): + return None + + data = json.loads(result.stdout) + creation_values = [] + for stream in data.get("streams") or []: + creation_values.append((stream.get("tags") or {}).get("creation_time")) + creation_values.append(((data.get("format") or {}).get("tags") or {}).get("creation_time")) + + for value in creation_values: + if not value: + continue + normalized = str(value).strip() + if normalized.endswith(("Z", "z")): + normalized = normalized[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(normalized) + if dt.tzinfo: + dt = dt.astimezone().replace(tzinfo=None) + return dt + except ValueError: + continue + except Exception: + return None + + return None + + def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Image] = None, extract_location_details: bool = True) -> Dict[str, Any]: """ Extracts photo_time, exif_info, and location from the file. Priority: - 1. EXIF DateTimeOriginal - 2. Filename (YYYYMMDD_HHMMSS or YYYYMMDD) - 3. Current Time + 1. EXIF DateTimeOriginal / DateTimeDigitized / DateTime + 2. Video metadata creation_time if ffprobe is available + 3. Filename date + 4. Folder date + 5. File modification time + 6. Current time """ metadata = { "photo_time": None, + "photo_time_source": None, "exif_info": None, "location": None, "width": None, @@ -186,7 +322,7 @@ def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Im # 1. Try EXIF try: - if file_path.lower().endswith(('.jpg', '.jpeg', '.tiff', '.webp', '.png', '.heic', '.heif')): + if file_path.lower().endswith(IMAGE_EXTENSIONS): exif_dict = None img = None should_close = False @@ -209,13 +345,11 @@ def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Im metadata["exif_info"] = exif_dict - # Extract Date (DateTimeOriginal) - date_str = exif_dict.get("DateTimeOriginal") - if date_str: - try: - metadata["photo_time"] = datetime.strptime(date_str, '%Y:%m:%d %H:%M:%S') - except ValueError: - pass + # Extract Date + photo_time, source = extract_datetime_from_exif(exif_dict) + if photo_time: + metadata["photo_time"] = photo_time + metadata["photo_time_source"] = source # Extract GPS gps = get_gps_info(exif_dict) @@ -245,21 +379,45 @@ def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Im print(traceback.format_exc()) print(f"Error extracting metadata: {e}") - # 2. If photo_time is still None, try Filename + # 2. If photo_time is still None, try video container metadata + if metadata["photo_time"] is None and file_path.lower().endswith(VIDEO_EXTENSIONS): + try: + photo_time = extract_datetime_from_video_metadata(file_path) + if photo_time: + metadata["photo_time"] = photo_time + metadata["photo_time_source"] = "video_metadata:creation_time" + except Exception: + pass + + # 3. If photo_time is still None, try Filename if metadata["photo_time"] is None: try: photo_time = extract_datetime_from_filename(filename) if photo_time: metadata["photo_time"] = photo_time - else: - photo_time = get_file_time_form_system(file_path) + metadata["photo_time_source"] = "filename" + except Exception: + pass + + # 4. If photo_time is still None, try dated folders + if metadata["photo_time"] is None: + try: + photo_time = extract_datetime_from_path(file_path) + if photo_time: metadata["photo_time"] = photo_time + metadata["photo_time_source"] = "path" except Exception: pass - # 3. Fallback to current time + # 5. Fallback to filesystem mtime + if metadata["photo_time"] is None: + photo_time = get_file_time_form_system(file_path) + metadata["photo_time"] = photo_time + metadata["photo_time_source"] = "file_mtime" + + # 6. Fallback to current time if metadata["photo_time"] is None: metadata["photo_time"] = datetime.now() + metadata["photo_time_source"] = "now" return metadata - diff --git a/package/server/app/utils/filename.py b/package/server/app/utils/filename.py index 0215cc36..5e8ce851 100644 --- a/package/server/app/utils/filename.py +++ b/package/server/app/utils/filename.py @@ -85,6 +85,8 @@ def _extract_datetime_from_filename(filename)-> datetime | None: r"(\d{8})[ _\.-](\d{2})[ _\.-](\d{2})[ _\.-](\d{2})", # 格式：YYYYMMDD_HH-MM-SS r"(\d{8})[ _T\.-](\d{6})", # 格式：YYYYMMDD_HHMMSS r"(\d{14})", # 格式：YYYYMMDDHHMMSS + r"(? datetime | None: return dt else: return is_valid_timestamp(timestamp, filename) + elif len(match.groups()) == 3: # 格式：YYYY-MM-DD / YYYYMMDD + year, month, day = match.groups() + dt = datetime(int(year), int(month), int(day)) + return dt elif len(match.groups()) == 4: # 格式：YYYY-MM-DD HHMMSS year, month, day, time_str = match.groups() if len(year) == 4: @@ -153,4 +159,4 @@ def extract_datetime_from_filename(filename) -> datetime | None: for filename in test_filenames: dt = extract_datetime_from_filename(filename) - print(f"Filename: {filename} -> Extracted Datetime: {dt}") \ No newline at end of file + print(f"Filename: {filename} -> Extracted Datetime: {dt}") diff --git a/package/server/tests/test_photo_time_extraction.py b/package/server/tests/test_photo_time_extraction.py new file mode 100644 index 00000000..88180c88 --- /dev/null +++ b/package/server/tests/test_photo_time_extraction.py @@ -0,0 +1,33 @@ +from datetime import datetime +import unittest + +from app.utils.exif import extract_datetime_from_path, parse_datetime_value +from app.utils.filename import extract_datetime_from_filename + + +class PhotoTimeExtractionTest(unittest.TestCase): + def test_extract_date_only_from_filename(self): + self.assertEqual(extract_datetime_from_filename("20250227").date(), datetime(2025, 2, 27).date()) + self.assertEqual(extract_datetime_from_filename("album 2024-3-28").date(), datetime(2024, 3, 28).date()) + + def test_extract_date_from_dated_folder(self): + path = "/app/Photos/20250227/DSC_1991.JPG" + + self.assertEqual(extract_datetime_from_path(path), datetime(2025, 2, 27)) + + def test_extract_inherited_year_month_day_from_folder(self): + path = "/app/Photos/2020.09.13罗文琪&文晶photo/9.15/jpg/1A5A8450z.jpg" + + self.assertEqual(extract_datetime_from_path(path), datetime(2020, 9, 15)) + + def test_extract_short_year_folder_date(self): + path = "/app/Photos/文晶写真照片/211004大唐芙蓉园（文女士）/DSC09505.JPG" + + self.assertEqual(extract_datetime_from_path(path), datetime(2021, 10, 4)) + + def test_parse_exif_datetime_fallback_value(self): + self.assertEqual(parse_datetime_value("2024:10:28 09:24:47"), datetime(2024, 10, 28, 9, 24, 47)) + + +if __name__ == "__main__": + unittest.main() From 85a5fd572a1ef595b33c3554fa279a7be08f73c7 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Fri, 29 May 2026 19:03:23 +0800 Subject: [PATCH 13/16] chore: mount Synology Photos libraries on NAS --- docker-compose.nas.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml index d4dc3154..6cfca0af 100644 --- a/docker-compose.nas.yml +++ b/docker-compose.nas.yml @@ -37,6 +37,8 @@ services: - ./package/server/main.py:/app/main.py:ro - ./package/server/app:/app/app:ro - /volume1/photo:/app/Photos/ + - /volume1/homes/rowankid/Photos:/app/SynologyPhotos/rowankid:ro + - /volume1/homes/wenjing/Photos:/app/SynologyPhotos/wenjing:ro environment: - TZ=Asia/Shanghai - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap From 3d5bf467428aef809c514cca7670c15f80210969 Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Fri, 29 May 2026 19:32:48 +0800 Subject: [PATCH 14/16] fix: skip Synology metadata directories during scan --- package/server/app/service/tasks/scan.py | 6 +++++- package/server/tests/test_scan_filters.py | 24 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 package/server/tests/test_scan_filters.py diff --git a/package/server/app/service/tasks/scan.py b/package/server/app/service/tasks/scan.py index 9558a8ab..4debd0b8 100644 --- a/package/server/app/service/tasks/scan.py +++ b/package/server/app/service/tasks/scan.py @@ -15,6 +15,8 @@ from app.core.config_manager import config_manager from app.service.live_photo import live_photo_service +IGNORED_SCAN_DIRS = {'@eaDir', '#recycle'} + def scan_directory_recursive(path: str, exts: Set[str], filter_settings: Optional[Dict] = None) -> Set[str]: found = set() try: @@ -47,6 +49,8 @@ def scan_directory_recursive(path: str, exts: Set[str], filter_settings: Optiona found.add(entry.path) elif entry.is_dir(): + if entry.name in IGNORED_SCAN_DIRS: + continue found.update(scan_directory_recursive(entry.path, exts, filter_settings)) except OSError: pass @@ -109,7 +113,7 @@ def parallel_scan_wrapper(): try: with os.scandir(root) as it: for entry in it: - if entry.is_dir(): + if entry.is_dir() and entry.name not in IGNORED_SCAN_DIRS: work_items.append(entry.path) except OSError: pass diff --git a/package/server/tests/test_scan_filters.py b/package/server/tests/test_scan_filters.py new file mode 100644 index 00000000..2ef1d5df --- /dev/null +++ b/package/server/tests/test_scan_filters.py @@ -0,0 +1,24 @@ +import os +import tempfile +import unittest + +from app.service.tasks.scan import scan_directory_recursive + + +class ScanFiltersTest(unittest.TestCase): + def test_scan_skips_synology_eadir_metadata(self): + with tempfile.TemporaryDirectory() as tmp: + real_photo = os.path.join(tmp, "IMG_0001.JPG") + os.makedirs(os.path.join(tmp, "@eaDir", "IMG_0001.JPG")) + metadata_thumb = os.path.join(tmp, "@eaDir", "IMG_0001.JPG", "SYNOPHOTO_THUMB_M.jpg") + + open(real_photo, "wb").close() + open(metadata_thumb, "wb").close() + + found = scan_directory_recursive(tmp, {".jpg", ".jpeg"}) + + self.assertEqual(found, {real_photo}) + + +if __name__ == "__main__": + unittest.main() From 0f9c63654c81fcad9b802b160a280ce81a151a0d Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Mon, 1 Jun 2026 09:10:49 +0800 Subject: [PATCH 15/16] Fix HEIC media file preview --- package/server/app/api/media.py | 11 +++-- package/server/tests/test_media_file.py | 58 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 package/server/tests/test_media_file.py diff --git a/package/server/app/api/media.py b/package/server/app/api/media.py index fa01a3cd..4206e761 100644 --- a/package/server/app/api/media.py +++ b/package/server/app/api/media.py @@ -154,10 +154,15 @@ async def get_media_file( raise HTTPException(status_code=404, detail="File not found") file_path = photo.file_path - # Determine media type + # Determine media type. Browsers cannot render HEIC/HEIF reliably, so serve + # the generated medium preview for the "file" view used by the lightbox. ext = os.path.splitext(file_path)[1].lower() - if ext == '.heic': - file_path = _get_thumbnail_path, photo.owner_id, photo_id, db, 'medium' + if ext in ('.heic', '.heif'): + file_path = await run_in_threadpool(_get_thumbnail_path, photo.owner_id, photo_id, db, 'medium') + preview_exists = await run_in_threadpool(os.path.exists, file_path) + if not preview_exists: + raise HTTPException(status_code=404, detail="Preview not found") + ext = os.path.splitext(file_path)[1].lower() file_size = await run_in_threadpool(os.path.getsize, file_path) media_type = "application/octet-stream" diff --git a/package/server/tests/test_media_file.py b/package/server/tests/test_media_file.py new file mode 100644 index 00000000..f4a51aa2 --- /dev/null +++ b/package/server/tests/test_media_file.py @@ -0,0 +1,58 @@ +import asyncio +import unittest +import uuid +from unittest.mock import ANY, patch + +from starlette.responses import FileResponse + +from app.api import media + + +class _FakePhoto: + def __init__(self, photo_id): + self.id = photo_id + self.owner_id = uuid.uuid4() + self.file_path = "/app/SynologyPhotos/rowankid/IMG_0001.HEIC" + + +class _FakeQuery: + def __init__(self, photo): + self.photo = photo + + def filter(self, *_args, **_kwargs): + return self + + def first(self): + return self.photo + + +class _FakeDb: + def __init__(self, photo): + self.photo = photo + + def query(self, _model): + return _FakeQuery(self.photo) + + +class MediaFileTest(unittest.TestCase): + def test_heic_file_endpoint_serves_medium_preview(self): + photo_id = uuid.uuid4() + photo = _FakePhoto(photo_id) + db = _FakeDb(photo) + preview_path = "/app/data/thumbnails/aa/bb/preview.webp" + + with ( + patch.object(media, "_get_thumbnail_path", return_value=preview_path) as get_thumbnail_path, + patch.object(media.os.path, "exists", return_value=True), + patch.object(media.os.path, "getsize", return_value=123), + ): + response = asyncio.run(media.get_media_file(photo_id, request=None, range=None, db=db)) + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.path, preview_path) + self.assertEqual(response.media_type, "image/webp") + get_thumbnail_path.assert_called_once_with(photo.owner_id, photo_id, ANY, "medium") + + +if __name__ == "__main__": + unittest.main() From 444544f5dc88abf733d3dc48d20083311d83ca5b Mon Sep 17 00:00:00 2001 From: rowankid <495872803@qq.com> Date: Tue, 2 Jun 2026 21:29:47 +0800 Subject: [PATCH 16/16] Tune AI task batch sizes --- package/server/app/service/task_worker.py | 4 +++- .../server/app/service/tasks/classification.py | 3 ++- package/server/app/service/tasks/face.py | 5 +++-- .../server/tests/test_task_worker_chunking.py | 16 ++++++++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 package/server/tests/test_task_worker_chunking.py diff --git a/package/server/app/service/task_worker.py b/package/server/app/service/task_worker.py index 8fb13865..f60f063d 100644 --- a/package/server/app/service/task_worker.py +++ b/package/server/app/service/task_worker.py @@ -72,12 +72,14 @@ def get_chunk_size(task_type): chunk_size = 4 if task_type == TaskType.PROCESS_BASIC: chunk_size = 1 + elif task_type == TaskType.RECOGNIZE_FACE: + chunk_size = 1 elif task_type == TaskType.VISUAL_DESCRIPTION: chunk_size = 2 elif task_type == TaskType.EXTRACT_METADATA: chunk_size = 16 elif task_type == TaskType.CLASSIFY_IMAGE: - chunk_size = 8 + chunk_size = 2 elif task_type == TaskType.IMAGE_EMBEDDING: chunk_size = 8 return chunk_size diff --git a/package/server/app/service/tasks/classification.py b/package/server/app/service/tasks/classification.py index efca8651..dfc80e17 100644 --- a/package/server/app/service/tasks/classification.py +++ b/package/server/app/service/tasks/classification.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) _tag_cache: Dict[str, str] = {} +CLASSIFICATION_AI_TIMEOUT = aiohttp.ClientTimeout(total=180) def get_tag_id(db: Session, tag_name: str, owner_id: Optional[UUID] = None) -> str: @@ -174,7 +175,7 @@ async def _process_owner_batch(self, owner_id: int, tasks: List[Task], db: Sessi return results # logger.info(f"valid tasks num: {len(valid_tasks)}") api_url = f"{config_manager.get_user_config(owner_id, db).ai.ai_api_url}/classification/" - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=CLASSIFICATION_AI_TIMEOUT) as session: async with session.post(api_url, json={"images": b64_images}) as resp: if resp.status == 200: result_data = await resp.json() diff --git a/package/server/app/service/tasks/face.py b/package/server/app/service/tasks/face.py index fea52887..6b4dc237 100644 --- a/package/server/app/service/tasks/face.py +++ b/package/server/app/service/tasks/face.py @@ -15,6 +15,7 @@ from app.service import storage logger = logging.getLogger(__name__) +FACE_AI_TIMEOUT = aiohttp.ClientTimeout(total=180) @TaskStrategyFactory.register(TaskType.RECOGNIZE_FACE) class RecognizeFaceStrategy(BaseTaskStrategy): @@ -178,7 +179,7 @@ async def process_batch(self, worker, tasks: List[Task], db: Session) -> List[Di # Batch AI request api_url = f"{config_manager.get_user_config(owner_id, db).ai.ai_api_url}/face/face-recognition" - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=FACE_AI_TIMEOUT) as session: async with session.post(api_url, json={"images": b64_images}) as resp: if resp.status == 200: result_data = await resp.json() @@ -261,7 +262,7 @@ async def process_single_photo(self, worker, photo: Photo, db: Session) -> Dict[ if not target_path or not os.path.exists(target_path): return {'status': 'failed', 'error': 'file not found'} - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=FACE_AI_TIMEOUT) as session: with open(target_path, 'rb') as f: file_data = f.read() width, height, _ = storage.get_image_dimensions(target_path) diff --git a/package/server/tests/test_task_worker_chunking.py b/package/server/tests/test_task_worker_chunking.py new file mode 100644 index 00000000..acb5f580 --- /dev/null +++ b/package/server/tests/test_task_worker_chunking.py @@ -0,0 +1,16 @@ +import unittest + +from app.db.models.task import TaskType +from app.service.task_worker import get_chunk_size + + +class TaskWorkerChunkingTest(unittest.TestCase): + def test_face_recognition_uses_single_image_batches(self): + self.assertEqual(get_chunk_size(TaskType.RECOGNIZE_FACE), 1) + + def test_classification_uses_small_batches(self): + self.assertEqual(get_chunk_size(TaskType.CLASSIFY_IMAGE), 2) + + +if __name__ == "__main__": + unittest.main()