diff --git a/.gitignore b/.gitignore index b6b1caa8..22e5878c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,7 @@ dist .trae *uploads* data/* +pg_data/ +backups/ .env -.claude/ \ No newline at end of file +.claude/ diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 00000000..1c9262af --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,81 @@ +services: + postgres: + image: pgvector/pgvector:pg18-trixie + container_name: trailsnap_postgres + restart: unless-stopped + environment: + TZ: Asia/Shanghai + POSTGRES_DB: trailsnap + POSTGRES_USER: trailsnap + POSTGRES_PASSWORD: trailsnap + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + PGDATA: /var/lib/postgresql/data/pgdata + networks: + - app-network + ports: + - "5532:5432" + volumes: + - trailsnap_pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U trailsnap -d trailsnap -p 5432"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s + + server: + image: siyuan044/trailsnap-server:latest + restart: unless-stopped + expose: + - "8000" + ports: + - "8800:8000" + networks: + - app-network + volumes: + - ./data:/app/data + - ./package/server/app:/app/app:ro + - /Volumes/photo:/app/Photos/:ro + environment: + - TZ=Asia/Shanghai + - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap + - RAILWAY_DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/railway + - AI_API_URL=http://ai:8001 + depends_on: + postgres: + condition: service_healthy + restart: true + + ai: + image: siyuan044/trailsnap-ai:latest + restart: unless-stopped + expose: + - "8001" + ports: + - "8801:8001" + networks: + - app-network + volumes: + - ./data:/app/data + environment: + - TZ=Asia/Shanghai + + frontend: + image: siyuan044/trailsnap-frontend:latest + restart: unless-stopped + ports: + - "8082:80" + depends_on: + - server + networks: + - app-network + environment: + - TZ=Asia/Shanghai + +networks: + app-network: + driver: bridge + +volumes: + trailsnap_pg_data: + external: true diff --git a/docker-compose.nas.yml b/docker-compose.nas.yml new file mode 100644 index 00000000..6cfca0af --- /dev/null +++ b/docker-compose.nas.yml @@ -0,0 +1,85 @@ +services: + postgres: + image: pgvector/pgvector:pg18-trixie + container_name: trailsnap_postgres + restart: unless-stopped + environment: + TZ: Asia/Shanghai + POSTGRES_DB: trailsnap + POSTGRES_USER: trailsnap + POSTGRES_PASSWORD: trailsnap + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + PGDATA: /var/lib/postgresql/data/pgdata + networks: + - app-network + ports: + - "5532:5432" + volumes: + - trailsnap_pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U trailsnap -d trailsnap -p 5432"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s + + server: + image: siyuan044/trailsnap-server:latest + restart: unless-stopped + expose: + - "8000" + ports: + - "8800:8000" + networks: + - app-network + volumes: + - ./data:/app/data + - ./package/server/main.py:/app/main.py:ro + - ./package/server/app:/app/app:ro + - /volume1/photo:/app/Photos/ + - /volume1/homes/rowankid/Photos:/app/SynologyPhotos/rowankid:ro + - /volume1/homes/wenjing/Photos:/app/SynologyPhotos/wenjing:ro + environment: + - TZ=Asia/Shanghai + - DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/trailsnap + - RAILWAY_DB_URL=postgresql://trailsnap:trailsnap@postgres:5432/railway + - AI_API_URL=http://192.168.10.12:8801 + depends_on: + postgres: + condition: service_healthy + restart: true + + ai: + image: siyuan044/trailsnap-ai:latest + restart: unless-stopped + expose: + - "8001" + ports: + - "8801:8001" + networks: + - app-network + volumes: + - ./data:/app/data + environment: + - TZ=Asia/Shanghai + + frontend: + image: siyuan044/trailsnap-frontend:latest + restart: unless-stopped + ports: + - "8082:80" + volumes: + - ./package/website/dist:/usr/share/nginx/html:ro + depends_on: + - server + networks: + - app-network + environment: + - TZ=Asia/Shanghai + +networks: + app-network: + driver: bridge + +volumes: + trailsnap_pg_data: diff --git a/package/server/app/api/classification.py b/package/server/app/api/classification.py index fc807867..d30f4f78 100644 --- a/package/server/app/api/classification.py +++ b/package/server/app/api/classification.py @@ -16,6 +16,13 @@ class RemovePhotosRequest(BaseModel): photo_ids: List[UUID] +class RenameTagRequest(BaseModel): + new_name: str + +class MergeTagsRequest(BaseModel): + target_name: str + source_names: List[str] + @router.get("", response_model=BaseResponse[List[schemas.TagStats]], summary="获取智能分类标签列表") def get_tags( skip: int = 0, @@ -29,6 +36,53 @@ def get_tags( data = crud.get_tags_with_stats(db, current_user.id, skip, limit) return BaseResponse(data=data) +@router.post("/merge", summary="合并分类标签") +def merge_tags( + payload: MergeTagsRequest = Body(..., description="合并标签请求"), + db: Session = Depends(get_db), + current_user: User = Depends(deps.get_current_user), +): + success, count, message = crud.merge_tags( + db, + current_user.id, + payload.target_name, + payload.source_names, + ) + if not success: + raise HTTPException(status_code=400, detail=message) + + from app.crud.album import trigger_conditional_albums_update + trigger_conditional_albums_update(db, current_user.id, None) + return BaseResponse(data={"status": "success", "count": count}) + +@router.put("/{path:path}", summary="重命名分类标签") +def rename_tag( + payload: RenameTagRequest = Body(..., description="标签重命名请求"), + path: str = Path(..., description="标签名称（支持多级/包含/）", path=True), + db: Session = Depends(get_db), + current_user: User = Depends(deps.get_current_user), +): + success, message = crud.rename_tag(db, current_user.id, path, payload.new_name) + if not success: + raise HTTPException(status_code=400, detail=message) + + from app.crud.album import trigger_conditional_albums_update + trigger_conditional_albums_update(db, current_user.id, None) + return BaseResponse(data={"status": "success"}) + +@router.delete("/{path:path}", summary="删除分类标签") +def delete_tag( + path: str = Path(..., description="标签名称（支持多级/包含/）", path=True), + db: Session = Depends(get_db), + current_user: User = Depends(deps.get_current_user), +): + if not crud.delete_tag(db, current_user.id, path): + raise HTTPException(status_code=404, detail="Tag not found") + + from app.crud.album import trigger_conditional_albums_update + trigger_conditional_albums_update(db, current_user.id, None) + return BaseResponse(data={"status": "success"}) + @router.get("/{path:path}/photos", response_model=BaseResponse[List[photo_schemas.Photo]], summary="获取分类照片列表") def get_tag_photos( # path=True 声明：匹配剩余的全部路径（支持包含/） diff --git a/package/server/app/api/media.py b/package/server/app/api/media.py index fa01a3cd..4206e761 100644 --- a/package/server/app/api/media.py +++ b/package/server/app/api/media.py @@ -154,10 +154,15 @@ async def get_media_file( raise HTTPException(status_code=404, detail="File not found") file_path = photo.file_path - # Determine media type + # Determine media type. Browsers cannot render HEIC/HEIF reliably, so serve + # the generated medium preview for the "file" view used by the lightbox. ext = os.path.splitext(file_path)[1].lower() - if ext == '.heic': - file_path = _get_thumbnail_path, photo.owner_id, photo_id, db, 'medium' + if ext in ('.heic', '.heif'): + file_path = await run_in_threadpool(_get_thumbnail_path, photo.owner_id, photo_id, db, 'medium') + preview_exists = await run_in_threadpool(os.path.exists, file_path) + if not preview_exists: + raise HTTPException(status_code=404, detail="Preview not found") + ext = os.path.splitext(file_path)[1].lower() file_size = await run_in_threadpool(os.path.getsize, file_path) media_type = "application/octet-stream" diff --git a/package/server/app/api/toolbox.py b/package/server/app/api/toolbox.py index 13a2e0c9..31e40e06 100644 --- a/package/server/app/api/toolbox.py +++ b/package/server/app/api/toolbox.py @@ -129,7 +129,7 @@ def get_duplicate_photos( if md5 not in grouped_photos: grouped_photos[md5] = [] # Convert DB model to Schema dict/object. We can just use the Pydantic model dump - # Fastapi will automatically serialize Pydantic models returned in a list/dict, + # Fastapi will automatically serialize Pydantic models returned in a list/dict, # but here we construct the dict structure manually. grouped_photos[md5].append(photo) @@ -173,7 +173,7 @@ def get_latest_similar_task( db, TaskType.SIMILAR_PHOTO_CLUSTERING, current_user.id, [TaskStatus.PENDING.value, TaskStatus.PROCESSING.value] ) - + if task: return BaseResponse(data=task) @@ -192,7 +192,7 @@ def get_latest_similar_task( task_id_uuid = UUID(latest_cluster.task_id) except ValueError: return BaseResponse(data=None) - + return BaseResponse(data=TaskResponse( id=task_id_uuid, type=TaskType.SIMILAR_PHOTO_CLUSTERING, @@ -203,7 +203,7 @@ def get_latest_similar_task( processed_items=0, result=None )) - + return BaseResponse(data=None) @router.get("/similar/tasks/{task_id}", response_model=BaseResponse[TaskResponse]) @@ -216,10 +216,10 @@ def get_similar_task( Get the status of a specific similar photo clustering task """ task = crud_task.get_task_by_id_and_owner(db, task_id, current_user.id) - + if task: return BaseResponse(data=task) - + # If not in Task table, it was either completed or deleted. # We assume it's completed. return BaseResponse(data=TaskResponse( @@ -247,17 +247,17 @@ def get_similar_task_result( result = [] current_skip = skip # Safety break to avoid infinite loops - max_loops = 100 + max_loops = 100 loop_count = 0 - + while len(result) < limit and loop_count < max_loops: loop_count += 1 - + # Calculate how many more we need remaining_needed = limit - len(result) # Fetch at least 20 or remaining_needed to be efficient fetch_limit = max(remaining_needed, 20) - + clusters = db.query(ImageCluster).filter( ImageCluster.task_id == str(task_id), ImageCluster.cluster_type == "SIMILARITY" @@ -268,21 +268,21 @@ def get_similar_task_result( processed_count = 0 deleted_count = 0 - + for cluster in clusters: # If we have enough results, we stop adding to result, # but we simply break and let the offset calculation handle the next page start. if len(result) >= limit: break - + processed_count += 1 - + photo_clusters = db.query(PhotoCluster).filter(PhotoCluster.cluster_id == cluster.cluster_id).all() photo_ids = [pc.photo_id for pc in photo_clusters] should_delete = False cluster_photos = [] - + if not photo_ids: should_delete = True else: @@ -297,24 +297,24 @@ def get_similar_task_result( score = 0 if desc: score = (desc.memory_score or 0) + (desc.quality_score or 0) - + cluster_photos.append((photo, score)) - + # Sort by score desc, then photo_time desc cluster_photos.sort(key=lambda x: (x[1], x[0].photo_time or datetime.min), reverse=True) - + if len(cluster_photos) < 2: should_delete = True - + if should_delete: # Delete invalid cluster db.delete(cluster) # Commit to ensure DB state reflects deletion for next query or consistency - db.commit() + db.commit() deleted_count += 1 else: result.append([x[0] for x in cluster_photos]) - + # Update current_skip for the next iteration or next page logic # logic: we advanced 'processed_count' positions in the original list, # but 'deleted_count' items were removed, so the DB shifts. @@ -333,11 +333,11 @@ def cancel_similar_task( Cancel/Delete a similar photo clustering task """ task = crud_task.get_task_by_id_and_owner(db, task_id, current_user.id) - + if task: if task.status in [TaskStatus.PENDING, TaskStatus.PROCESSING]: task.status = TaskStatus.CANCELLED - # Note: This doesn't stop the running thread immediately if it's processing, + # Note: This doesn't stop the running thread immediately if it's processing, # but TaskWorker should handle cancellation check. crud_task.delete_task(db, task) else: @@ -347,7 +347,7 @@ def cancel_similar_task( return BaseResponse(code=404, msg="Task not found", data=None) for cluster in clusters: db.delete(cluster) - + db.commit() return BaseResponse(data={"message": "Task deleted"}) @@ -369,17 +369,34 @@ def get_photos_for_cleanup( db: Session = Depends(get_db), current_user: User = Depends(get_current_user) ): - # Join with ImageDescription to access scores - query = db.query(Photo).join(ImageDescriptionModel, Photo.id == ImageDescriptionModel.photo_id).filter(Photo.owner_id == current_user.id, Photo.is_deleted == False) - - # Calculate score: memory_score + quality_score - # We use coalesce to treat nulls as 0 - score_expr = func.coalesce(ImageDescriptionModel.memory_score, 0) + func.coalesce(ImageDescriptionModel.quality_score, 0) + score_subquery = db.query( + ImageDescriptionModel.photo_id.label("photo_id"), + func.max( + func.coalesce(ImageDescriptionModel.memory_score, 0) + + func.coalesce(ImageDescriptionModel.quality_score, 0) + ).label("score") + ).group_by(ImageDescriptionModel.photo_id).subquery() + + query = db.query(Photo).join( + score_subquery, + Photo.id == score_subquery.c.photo_id + ).filter( + Photo.owner_id == current_user.id, + Photo.is_deleted == False + ) if sort_by == "desc": - query = query.order_by(score_expr.desc()) + query = query.order_by( + score_subquery.c.score.desc(), + Photo.photo_time.desc().nulls_last(), + Photo.id.desc(), + ) else: - query = query.order_by(score_expr.asc()) + query = query.order_by( + score_subquery.c.score.asc(), + Photo.photo_time.asc().nulls_last(), + Photo.id.asc(), + ) photos = query.offset(skip).limit(limit).all() return BaseResponse(data=photos) @@ -424,28 +441,28 @@ def get_organize_preview_options( from app.db.models.photo_metadata import PhotoMetadata from sqlalchemy import and_, or_ import os - + options = set() - + if req.strategy == 'category': tags = db.query(PhotoTag.tag_name)\ .join(PhotoTagRelation, PhotoTag.id == PhotoTagRelation.tag_id)\ .join(Photo, PhotoTagRelation.photo_id == Photo.id)\ .filter(Photo.owner_id == current_user.id, Photo.is_deleted.is_(False))\ .distinct().all() - + for (tag_name,) in tags: if tag_name: options.add(tag_name) - + has_untagged = db.query(Photo.id)\ .outerjoin(PhotoTagRelation, Photo.id == PhotoTagRelation.photo_id)\ .filter( - Photo.owner_id == current_user.id, + Photo.owner_id == current_user.id, Photo.is_deleted.is_(False), PhotoTagRelation.tag_id.is_(None) ).first() - + if has_untagged: options.add('未分类') @@ -454,26 +471,26 @@ def get_organize_preview_options( .join(Face, FaceIdentity.id == Face.face_identity_id)\ .join(Photo, Face.photo_id == Photo.id)\ .filter( - Photo.owner_id == current_user.id, + Photo.owner_id == current_user.id, Photo.is_deleted.is_(False), FaceIdentity.identity_name.isnot(None), FaceIdentity.identity_name != '' ).distinct().all() - + for (identity_name,) in identities: options.add(identity_name) - + valid_photo_ids_query = db.query(Face.photo_id)\ .join(FaceIdentity, Face.face_identity_id == FaceIdentity.id)\ .filter(FaceIdentity.identity_name.isnot(None), FaceIdentity.identity_name != '') - + has_unnamed = db.query(Photo.id)\ .filter( - Photo.owner_id == current_user.id, + Photo.owner_id == current_user.id, Photo.is_deleted.is_(False), Photo.id.notin_(valid_photo_ids_query) ).first() - + if has_unnamed: options.add('未命名') @@ -482,11 +499,11 @@ def get_organize_preview_options( .join(Photo, PhotoMetadata.photo_id == Photo.id)\ .filter(Photo.owner_id == current_user.id, Photo.is_deleted.is_(False))\ .distinct().all() - + for prov, city, dist in locations: if not prov and not city and not dist: continue - + parts = [] if req.location_granularity == 'province' and prov: parts.append(prov) @@ -504,7 +521,7 @@ def get_organize_preview_options( if prov: parts.append(prov) if city and (not parts or parts[-1] != city): parts.append(city) if dist and (not parts or parts[-1] != dist): parts.append(dist) - + if not parts: options.add('未知位置') else: @@ -512,12 +529,12 @@ def get_organize_preview_options( options.add(os.path.join(*parts)) else: options.add("-".join(parts)) - + if '未知位置' not in options: has_unknown_location = db.query(Photo.id)\ .outerjoin(PhotoMetadata, Photo.id == PhotoMetadata.photo_id)\ .filter( - Photo.owner_id == current_user.id, + Photo.owner_id == current_user.id, Photo.is_deleted.is_(False), or_( PhotoMetadata.photo_id.is_(None), @@ -528,10 +545,10 @@ def get_organize_preview_options( ) ) ).first() - + if has_unknown_location: options.add('未知位置') - + return BaseResponse(data=OrganizePreviewOptionsResponse(options=list(options))) @router.get("/organize/tasks/latest", response_model=BaseResponse[Optional[TaskResponse]]) @@ -616,4 +633,3 @@ def get_latest_time_from_filename_task( [TaskStatus.PENDING.value, TaskStatus.PROCESSING.value, TaskStatus.COMPLETED.value, TaskStatus.FAILED.value] ) return BaseResponse(data=task) - diff --git a/package/server/app/crud/tag.py b/package/server/app/crud/tag.py index 7abfd59c..7882645f 100644 --- a/package/server/app/crud/tag.py +++ b/package/server/app/crud/tag.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List, Optional, Tuple from uuid import UUID from sqlalchemy.orm import Session from app.db.models.tag import PhotoTag, PhotoTagRelation @@ -184,3 +184,76 @@ def set_tag_cover(db: Session, owner_id: UUID, tag_name: str, photo_id: UUID) -> tag.cover_id = photo_id db.commit() return True + +def delete_tag(db: Session, owner_id: UUID, tag_name: str) -> bool: + tag = get_tag_by_name(db, tag_name, owner_id) + if not tag: + return False + + db.query(PhotoTagRelation).filter(PhotoTagRelation.tag_id == tag.id).delete(synchronize_session=False) + db.delete(tag) + db.commit() + return True + +def rename_tag(db: Session, owner_id: UUID, old_name: str, new_name: str) -> Tuple[bool, str]: + new_name = new_name.strip() + if not new_name: + return False, "New tag name is empty" + + tag = get_tag_by_name(db, old_name, owner_id) + if not tag: + return False, "Tag not found" + + existing = get_tag_by_name(db, new_name, owner_id) + if existing and existing.id != tag.id: + return False, "Target tag already exists" + + tag.tag_name = new_name + db.commit() + return True, "success" + +def merge_tags(db: Session, owner_id: UUID, target_name: str, source_names: List[str]) -> Tuple[bool, int, str]: + target_name = target_name.strip() + source_names = [name.strip() for name in source_names if name and name.strip()] + if not target_name: + return False, 0, "Target tag name is empty" + if not source_names: + return False, 0, "Source tags are empty" + + target = get_tag_by_name(db, target_name, owner_id) + if not target: + target = create_tag(db, target_name, "classification", owner_id) + + merged_count = 0 + for source_name in source_names: + if source_name == target_name: + continue + + source = get_tag_by_name(db, source_name, owner_id) + if not source: + continue + + if not target.cover_id and source.cover_id: + target.cover_id = source.cover_id + + source_relations = db.query(PhotoTagRelation).filter( + PhotoTagRelation.tag_id == source.id + ).all() + + for relation in source_relations: + target_relation = db.query(PhotoTagRelation).filter( + PhotoTagRelation.photo_id == relation.photo_id, + PhotoTagRelation.tag_id == target.id + ).first() + + if target_relation: + target_relation.confidence = max(target_relation.confidence or 0, relation.confidence or 0) + db.delete(relation) + else: + relation.tag_id = target.id + merged_count += 1 + + db.delete(source) + + db.commit() + return True, merged_count, "success" diff --git a/package/server/app/service/storage.py b/package/server/app/service/storage.py index 0ee067a1..f5b47acd 100644 --- a/package/server/app/service/storage.py +++ b/package/server/app/service/storage.py @@ -22,6 +22,11 @@ # Global cache for storage root (User ID -> Root Path) _STORAGE_ROOT_CACHE = {} +RAW_EXTENSIONS = { + '.arw', '.cr2', '.cr3', '.dng', '.nef', '.nrw', '.orf', + '.pef', '.raf', '.raw', '.rw2', '.srw', '.x3f' +} + def _get_storage_root(user_id: UUID, db: Session = None) -> str: try: root = './data/uploads' # Default path @@ -237,6 +242,67 @@ def get_live_photo_vide(image_path: str) -> Optional[str]: logging.error(f"Error getting live photo video for {image_path}: {e}") return None +def _raw_candidate_dirs(file_path: str): + parent = os.path.abspath(os.path.dirname(file_path)) + seen = set() + + def add(path: str): + path = os.path.abspath(path) + if path not in seen and os.path.isdir(path): + seen.add(path) + yield path + + yield from add(parent) + + def add_raw_dir_with_children(path: str): + if 'raw' not in os.path.basename(path).lower(): + return + yield from add(path) + try: + for name in os.listdir(path): + child = os.path.join(path, name) + if os.path.isdir(child): + yield from add(child) + except OSError as e: + logging.warning(f"Could not inspect RAW sidecar directory {path}: {e}") + + try: + for name in os.listdir(parent): + yield from add_raw_dir_with_children(os.path.join(parent, name)) + except OSError as e: + logging.warning(f"Could not inspect photo directory {parent}: {e}") + + grandparent = os.path.dirname(parent) + if grandparent and grandparent != parent: + try: + for name in os.listdir(grandparent): + yield from add_raw_dir_with_children(os.path.join(grandparent, name)) + except OSError as e: + logging.warning(f"Could not inspect sibling directories for {parent}: {e}") + +def delete_raw_sidecars(file_path: str): + base_name, ext = os.path.splitext(os.path.basename(file_path)) + if ext.lower() in RAW_EXTENSIONS: + return + + base_name_lower = base_name.lower() + for directory in _raw_candidate_dirs(file_path): + try: + names = os.listdir(directory) + except OSError as e: + logging.warning(f"Could not inspect RAW sidecar candidates in {directory}: {e}") + continue + + for name in names: + candidate_base, candidate_ext = os.path.splitext(name) + if candidate_base.lower() == base_name_lower and candidate_ext.lower() in RAW_EXTENSIONS: + candidate = os.path.join(directory, name) + try: + os.remove(candidate) + logging.info(f"Deleted RAW sidecar: {candidate}") + except OSError as e: + logging.error(f"Error deleting RAW sidecar {candidate}: {e}") + def delete_file(user_id: UUID, file_path: str, file_id: UUID, is_live_photo: bool = False): try: if os.path.exists(file_path): @@ -245,6 +311,7 @@ def delete_file(user_id: UUID, file_path: str, file_id: UUID, is_live_photo: boo video_path = get_live_photo_vide(file_path) if video_path and os.path.exists(video_path): os.remove(video_path) + delete_raw_sidecars(file_path) delete_thumbnails(user_id, file_id) except Exception as e: logging.error(f"Error deleting file {user_id}/{file_path}: {e}") diff --git a/package/server/app/service/task_worker.py b/package/server/app/service/task_worker.py index 41320a00..f60f063d 100644 --- a/package/server/app/service/task_worker.py +++ b/package/server/app/service/task_worker.py @@ -70,12 +70,16 @@ def task_done(self, category: str): def get_chunk_size(task_type): chunk_size = 4 - if task_type == TaskType.VISUAL_DESCRIPTION: + if task_type == TaskType.PROCESS_BASIC: + chunk_size = 1 + elif task_type == TaskType.RECOGNIZE_FACE: + chunk_size = 1 + elif task_type == TaskType.VISUAL_DESCRIPTION: chunk_size = 2 - elif task_type == TaskType.PROCESS_BASIC or task_type == TaskType.EXTRACT_METADATA: + elif task_type == TaskType.EXTRACT_METADATA: chunk_size = 16 elif task_type == TaskType.CLASSIFY_IMAGE: - chunk_size = 8 + chunk_size = 2 elif task_type == TaskType.IMAGE_EMBEDDING: chunk_size = 8 return chunk_size @@ -230,7 +234,7 @@ def check_task_for_release(self): last_run = self.last_active_time[task_type] if (datetime.now() - last_run).total_seconds() > 300: idle_types.append(task_type) - + if idle_types: TaskStrategyFactory.release_idle_resources(idle_types) for t in idle_types: @@ -275,7 +279,8 @@ def _manage_pool_lifecycle(self): if active_count > 0: if active_cpu_count > 0 and self.process_pool is None: logging.info(f"Restarting process pool") - self.process_pool = concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) + max_workers = max(1, system_config.config.task.max_concurrent_tasks) + self.process_pool = concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) if self.thread_pool is None and active_io_count > 0: max_workers = system_config.config.task.max_concurrent_tasks logging.info(f"Restarting thread pool") @@ -307,17 +312,19 @@ def _fetch_tasks_to_queues_sync(self, allowed_types: List[str], current_qsizes: lowest_priorities = {'CPU': -9999, 'IO': -9999, 'AI': -9999} try: # We will fetch up to max_batch_size per category if its queue is below threshold - # Max items in queue per category - QUEUE_THRESHOLD = 50 - # How many items to fetch in one DB query per category - FETCH_BATCH_SIZE = 48 + # Keep the local Docker setup gentle. The original queue sizes can + # mark hundreds of tasks as PROCESSING before they actually run, + # which is painful when NAS IO or Docker Desktop storage stalls. + max_tasks = max(1, system_config.config.task.max_concurrent_tasks) + QUEUE_THRESHOLD = max_tasks + FETCH_BATCH_SIZE = max_tasks type_conditions = [] for cat in ['CPU', 'IO', 'AI']: cat_types = [t for t in allowed_types if TaskStrategyFactory.get_strategy(t) and TaskStrategyFactory.get_strategy(t).task_category == cat] if not cat_types: continue - + qsize = current_qsizes.get(cat, 0) if qsize < QUEUE_THRESHOLD: # Queue not full, allow all priorities for these types @@ -342,7 +349,7 @@ def _fetch_tasks_to_queues_sync(self, allowed_types: List[str], current_qsizes: for task in tasks: cat = TaskStrategyFactory.get_strategy(task.type).task_category if not cat: continue - + if cat == 'AI' and task.owner_id: user_config = config_manager.get_user_config(task.owner_id, db) if user_config.ai.analysis_connection_id == 'builtin': @@ -377,12 +384,13 @@ async def consumer_loop(self, category: str): # Configure max concurrency per consumer category based on system settings # or Fast Mode. Using Semaphores to allow multiple batches to run concurrently. max_concurrency = 1 + configured_workers = max(1, system_config.config.task.max_concurrent_tasks) if category == 'CPU': - max_concurrency = os.cpu_count() or 4 + max_concurrency = configured_workers elif category == 'IO': - max_concurrency = 4 + max_concurrency = configured_workers elif category == 'AI': - max_concurrency = 1 + max_concurrency = configured_workers semaphore = asyncio.Semaphore(max_concurrency) @@ -417,7 +425,7 @@ async def wrapper(b): # 放开后台任务执行 asyncio.create_task(wrapper(batch)) - + except asyncio.CancelledError: break except Exception as e: @@ -680,4 +688,4 @@ def add_task(self, db: Session, type: str, payload: dict, priority: int = 0, own def add_tasks(self, db: Session, tasks_data: List[Dict], owner_id: UUID = None): """Batch add tasks""" - crud_task.add_tasks(db, tasks_data, owner_id) \ No newline at end of file + crud_task.add_tasks(db, tasks_data, owner_id) diff --git a/package/server/app/service/tasks/classification.py b/package/server/app/service/tasks/classification.py index efca8651..dfc80e17 100644 --- a/package/server/app/service/tasks/classification.py +++ b/package/server/app/service/tasks/classification.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) _tag_cache: Dict[str, str] = {} +CLASSIFICATION_AI_TIMEOUT = aiohttp.ClientTimeout(total=180) def get_tag_id(db: Session, tag_name: str, owner_id: Optional[UUID] = None) -> str: @@ -174,7 +175,7 @@ async def _process_owner_batch(self, owner_id: int, tasks: List[Task], db: Sessi return results # logger.info(f"valid tasks num: {len(valid_tasks)}") api_url = f"{config_manager.get_user_config(owner_id, db).ai.ai_api_url}/classification/" - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=CLASSIFICATION_AI_TIMEOUT) as session: async with session.post(api_url, json={"images": b64_images}) as resp: if resp.status == 200: result_data = await resp.json() diff --git a/package/server/app/service/tasks/face.py b/package/server/app/service/tasks/face.py index fea52887..6b4dc237 100644 --- a/package/server/app/service/tasks/face.py +++ b/package/server/app/service/tasks/face.py @@ -15,6 +15,7 @@ from app.service import storage logger = logging.getLogger(__name__) +FACE_AI_TIMEOUT = aiohttp.ClientTimeout(total=180) @TaskStrategyFactory.register(TaskType.RECOGNIZE_FACE) class RecognizeFaceStrategy(BaseTaskStrategy): @@ -178,7 +179,7 @@ async def process_batch(self, worker, tasks: List[Task], db: Session) -> List[Di # Batch AI request api_url = f"{config_manager.get_user_config(owner_id, db).ai.ai_api_url}/face/face-recognition" - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=FACE_AI_TIMEOUT) as session: async with session.post(api_url, json={"images": b64_images}) as resp: if resp.status == 200: result_data = await resp.json() @@ -261,7 +262,7 @@ async def process_single_photo(self, worker, photo: Photo, db: Session) -> Dict[ if not target_path or not os.path.exists(target_path): return {'status': 'failed', 'error': 'file not found'} - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=FACE_AI_TIMEOUT) as session: with open(target_path, 'rb') as f: file_data = f.read() width, height, _ = storage.get_image_dimensions(target_path) diff --git a/package/server/app/service/tasks/scan.py b/package/server/app/service/tasks/scan.py index 479b8474..14fdd489 100644 --- a/package/server/app/service/tasks/scan.py +++ b/package/server/app/service/tasks/scan.py @@ -15,6 +15,8 @@ from app.core.config_manager import config_manager from app.service.live_photo import live_photo_service +IGNORED_SCAN_DIRS = {'@eaDir', '#recycle'} + def scan_directory_recursive(path: str, exts: Set[str], filter_settings: Optional[Dict] = None) -> Set[str]: found = set() try: @@ -47,6 +49,8 @@ def scan_directory_recursive(path: str, exts: Set[str], filter_settings: Optiona found.add(entry.path) elif entry.is_dir(): + if entry.name in IGNORED_SCAN_DIRS: + continue found.update(scan_directory_recursive(entry.path, exts, filter_settings)) except OSError: pass @@ -81,10 +85,10 @@ async def process(self, worker, task: Task, db: Session): if not roots: config = config_manager.get_user_config(user.id, db) roots = config.storage.external_directories - + if not roots: continue - + res = await self._scan_for_user(worker, db, user, roots) total_new += res.get('new_files', 0) total_deleted += res.get('deleted_files', 0) @@ -109,7 +113,7 @@ def parallel_scan_wrapper(): try: with os.scandir(root) as it: for entry in it: - if entry.is_dir(): + if entry.is_dir() and entry.name not in IGNORED_SCAN_DIRS: work_items.append(entry.path) except OSError: pass @@ -123,7 +127,7 @@ def parallel_scan_wrapper(): files_on_disk = await loop.run_in_executor(None, parallel_scan_wrapper) existing_files, live_photo_to_add = self._get_existing_files(db, str(user.id), scan_roots) - + # Determine new and deleted new_files = files_on_disk - existing_files deleted_files = existing_files - files_on_disk @@ -181,7 +185,7 @@ def _get_existing_files(self, db: Session, user_id: str, scan_roots: List[str]) elif file_path.endswith('.mov') and (p[0][:-3] + 'HEIC' in existing_files): live_photo_to_add.add(p[0]) existing_files.add(p[0]) - + return existing_files, live_photo_to_add async def _create_tasks_for_new_files(self, user_id: str, new_files: Set[str], loop: asyncio.AbstractEventLoop, db: Session): @@ -264,7 +268,7 @@ def check_live_pair(img, vid): def _handle_deleted_files(self, user_id: str, deleted_files: Set[str], db: Session, worker): if not deleted_files: return - + deleted_list = list(deleted_files) chunk_size = 500 for i in range(0, len(deleted_list), chunk_size): @@ -274,11 +278,11 @@ def _handle_deleted_files(self, user_id: str, deleted_files: Set[str], db: Sessi for ph in photos_to_delete: photo_ids_to_delete.append(ph.id) db.add(IndexLog(action='deleted', file_path=ph.file_path, photo_id=ph.id, owner_id=user_id)) - + if photo_ids_to_delete: from app.crud.photo import batch_delete_photos_db batch_delete_photos_db(db, photo_ids_to_delete, is_delete_file=False, user_id=user_id) - + db.commit() worker.scan_status['deleted'] += len(photos_to_delete) diff --git a/package/server/app/service/tasks/visual_description.py b/package/server/app/service/tasks/visual_description.py index 7e240fd1..e1e0c475 100644 --- a/package/server/app/service/tasks/visual_description.py +++ b/package/server/app/service/tasks/visual_description.py @@ -5,9 +5,12 @@ import os import json import base64 +import re +import io from typing import Dict, Any, List from sqlalchemy.orm import Session from langchain_openai import ChatOpenAI +from PIL import Image from app.db.models import PhotoMetadata from app.db.models.task import Task, TaskType @@ -19,8 +22,94 @@ logger = logging.getLogger(__name__) -import io -from PIL import Image +def _extract_json_string_field(text: str, key: str) -> str | None: + match = re.search(rf'"{key}"\s*:\s*"', text) + if not match: + return None + + start = match.end() + next_key = re.search( + r',?\s*\n?\s*"(description|tags|memory_score|beauty_score|quality_score|reason|narrative)"\s*:', + text[start:], + ) + end = start + next_key.start() if next_key else len(text) + value = text[start:end].strip() + value = re.sub(r'"\s*,?\s*\}?\s*$', '', value, flags=re.DOTALL).strip() + return value.replace('\\"', '"') + + +def _extract_json_array_field(text: str, key: str) -> List[str] | None: + match = re.search(rf'"{key}"\s*:\s*(\[[^\]]*\])', text, re.DOTALL) + if not match: + return None + raw_value = match.group(1) + try: + value = json.loads(raw_value, strict=False) + if isinstance(value, list): + return value + except json.JSONDecodeError: + pass + return re.findall(r'"([^"]+)"', raw_value) + + +def _extract_json_number_field(text: str, key: str) -> float | None: + match = re.search(rf'"{key}"\s*:\s*(-?\d+(?:\.\d+)?)', text) + if not match: + return None + return float(match.group(1)) + + +def _parse_partial_json_response(text: str) -> Dict[str, Any]: + result = { + "description": _extract_json_string_field(text, "description"), + "tags": _extract_json_array_field(text, "tags") or [], + "memory_score": _extract_json_number_field(text, "memory_score"), + "beauty_score": _extract_json_number_field(text, "beauty_score"), + "quality_score": _extract_json_number_field(text, "quality_score"), + "reason": _extract_json_string_field(text, "reason"), + "narrative": _extract_json_string_field(text, "narrative"), + } + result = {key: value for key, value in result.items() if value not in (None, "")} + if "description" in result or "narrative" in result: + return result + raise ValueError("No usable fields found in partial JSON response") + + +def parse_json_response(content: str) -> Dict[str, Any]: + text = (content or "").strip() + if text.startswith("```"): + match = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL | re.IGNORECASE) + if match: + text = match.group(1).strip() + + candidates = [text] + brace_index = text.find("{") + if brace_index > 0: + candidates.append(text[brace_index:]) + + last_error = None + decoder = json.JSONDecoder(strict=False) + for candidate in candidates: + if not candidate: + continue + try: + result, _ = decoder.raw_decode(candidate.strip()) + if isinstance(result, dict): + return result + except json.JSONDecodeError as e: + last_error = e + + for candidate in candidates: + if not candidate: + continue + try: + return _parse_partial_json_response(candidate.strip()) + except ValueError: + pass + + preview = text[:300].replace("\n", "\\n") + raise ValueError(f"Failed to parse model JSON response: {preview}") from last_error + def encode_image(image_path, max_size=672): with Image.open(image_path) as img: @@ -46,6 +135,13 @@ def encode_image(image_path, max_size=672): return base64_str +def mark_visual_description_processed(photo: Photo, db: Session) -> None: + tasks_status = dict(photo.processed_tasks or {}) + tasks_status['visual_description'] = True + photo.processed_tasks = tasks_status + db.commit() + + @TaskStrategyFactory.register(TaskType.VISUAL_DESCRIPTION) class VisualDescriptionStrategy(BaseTaskStrategy): @property @@ -73,18 +169,23 @@ def create_client(self, settings): logger.error("Visual Model connection has no api_key: %s", settings.analysis_connection_id) raise ValueError(f"Visual Model connection has no api_key: {settings.analysis_connection_id}") - # 2. Call OpenAI API - client = ChatOpenAI( - api_key=connection.api_key, - model= settings.analysis_model_name, - base_url=connection.api_base if connection.api_base else None, - timeout=60, - max_completion_tokens=4096, - extra_body={ + # 2. Call OpenAI-compatible API. Some providers reject + # reasoning_effort="none"; disabling thinking through extra_body is + # enough for Xiaomi Mimo while still working with LM Studio Qwen. + client_kwargs = { + "api_key": connection.api_key, + "model": settings.analysis_model_name, + "base_url": connection.api_base if connection.api_base else None, + "timeout": 60, + "max_completion_tokens": 4096, + "extra_body": { "chat_template_kwargs": {"enable_thinking": False}, }, - reasoning_effort="none", - ) + } + if "xiaomimimo.com" not in (connection.api_base or ""): + client_kwargs["reasoning_effort"] = "none" + + client = ChatOpenAI(**client_kwargs) return client async def process(self, worker, task: Task, db: Session) -> Dict[str, Any]: @@ -120,7 +221,7 @@ async def process(self, worker, task: Task, db: Session) -> Dict[str, Any]: photo = db.query(Photo).filter(Photo.id == photo_id).first() if not photo: return {'status': 'skipped', 'reason': 'photo not found'} - + # Check if already processed (unless force) if not force: tasks_status = photo.processed_tasks or {} @@ -186,7 +287,7 @@ async def process_single_photo(self, worker, photo: Photo, db: Session, settings user_config = config_manager.get_user_config(photo.owner_id, db) settings = user_config.ai client =self.create_client(settings) - + target_path = storage.get_preview_path(photo.owner_id, photo.id) if not target_path or not os.path.exists(target_path): target_path = photo.file_path @@ -219,16 +320,14 @@ async def process_single_photo(self, worker, photo: Photo, db: Session, settings ] ) - eval_content = eval_response.content.strip().strip('`').strip().strip('json') - print(eval_content) - # Clean up code blocks if present - if eval_content.startswith("```"): - eval_content = eval_content.strip("`") - if eval_content.startswith("json"): - eval_content = eval_content[4:] + eval_content = eval_response.content + if "considered high risk" in (eval_content or "").lower(): + mark_visual_description_processed(photo, db) + return {'status': 'skipped', 'reason': 'model rejected high risk image'} + try: - result_json = json.loads(eval_content.strip()) - except json.JSONDecodeError as e: + result_json = parse_json_response(eval_content) + except (json.JSONDecodeError, ValueError) as e: logger.error(f"Failed to parse evaluation JSON for photo {photo.id}: {eval_content}") raise e @@ -251,10 +350,7 @@ async def process_single_photo(self, worker, photo: Photo, db: Session, settings ) db.add(desc) # Update photo processed status - tasks_status = dict(photo.processed_tasks or {}) - tasks_status['visual_description'] = True - photo.processed_tasks = tasks_status - db.commit() + mark_visual_description_processed(photo, db) return { 'status': 'completed', 'description': desc.description, diff --git a/package/server/app/utils/exif.py b/package/server/app/utils/exif.py index 2bb0b6c3..224b7bde 100644 --- a/package/server/app/utils/exif.py +++ b/package/server/app/utils/exif.py @@ -9,6 +9,7 @@ @Description : """ import shutil +import subprocess import traceback from datetime import datetime import re @@ -26,6 +27,9 @@ from app.utils.filename import extract_datetime_from_filename +IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.tiff', '.webp', '.png', '.heic', '.heif') +VIDEO_EXTENSIONS = ('.mp4', '.mov', '.avi', '.mkv', '.webm', '.m4v') + # Helper Functions for Metadata # resources/rg_data RG_DIR = os.path.join(os.path.dirname(__file__), '../../resources/rg_data') @@ -168,16 +172,148 @@ def get_file_time_form_system(file_path: str) -> datetime: return datetime.now() +def parse_datetime_value(value: Any) -> Optional[datetime]: + if not value: + return None + + if isinstance(value, (bytes, bytearray)): + value = value.decode(errors="ignore") + + value = str(value).strip().replace("\x00", "") + formats = ( + "%Y:%m:%d %H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y:%m:%d %H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S%z", + ) + for fmt in formats: + try: + dt = datetime.strptime(value, fmt) + if dt.tzinfo: + dt = dt.astimezone().replace(tzinfo=None) + return dt + except ValueError: + continue + return None + + +def extract_datetime_from_exif(exif_data: Dict[str, Any]) -> tuple[Optional[datetime], Optional[str]]: + for tag_name in ("DateTimeOriginal", "DateTimeDigitized", "DateTime"): + dt = parse_datetime_value(exif_data.get(tag_name)) + if dt: + return dt, f"exif:{tag_name}" + return None, None + + +def _date_from_short_token(text: str) -> Optional[datetime]: + match = re.search(r"(? Optional[datetime]: + if not year: + return None + match = re.search(r"(? Optional[datetime]: + """ + Recover a coarse capture date from dated folders, e.g. 20250227, + 2024-3-28, 211004..., or 2020.09.13/.../9.15. + """ + inherited_year = None + best_dt = None + directory = os.path.dirname(file_path) + for part in [p for p in directory.split(os.sep) if p]: + dt = extract_datetime_from_filename(part) or _date_from_short_token(part) + if dt: + inherited_year = dt.year + best_dt = dt + continue + + month_day_dt = _date_from_month_day(part, inherited_year) + if month_day_dt: + best_dt = month_day_dt + + return best_dt + + +def extract_datetime_from_video_metadata(file_path: str) -> Optional[datetime]: + if not shutil.which("ffprobe"): + return None + + try: + result = subprocess.run( + [ + "ffprobe", + "-v", + "quiet", + "-print_format", + "json", + "-show_entries", + "format_tags=creation_time:stream_tags=creation_time", + file_path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=12, + ) + if result.returncode != 0 or not result.stdout.strip(): + return None + + data = json.loads(result.stdout) + creation_values = [] + for stream in data.get("streams") or []: + creation_values.append((stream.get("tags") or {}).get("creation_time")) + creation_values.append(((data.get("format") or {}).get("tags") or {}).get("creation_time")) + + for value in creation_values: + if not value: + continue + normalized = str(value).strip() + if normalized.endswith(("Z", "z")): + normalized = normalized[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(normalized) + if dt.tzinfo: + dt = dt.astimezone().replace(tzinfo=None) + return dt + except ValueError: + continue + except Exception: + return None + + return None + + def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Image] = None, extract_location_details: bool = True) -> Dict[str, Any]: """ Extracts photo_time, exif_info, and location from the file. Priority: - 1. EXIF DateTimeOriginal - 2. Filename (YYYYMMDD_HHMMSS or YYYYMMDD) - 3. Current Time + 1. EXIF DateTimeOriginal / DateTimeDigitized / DateTime + 2. Video metadata creation_time if ffprobe is available + 3. Filename date + 4. Folder date + 5. File modification time + 6. Current time """ metadata = { "photo_time": None, + "photo_time_source": None, "exif_info": None, "location": None, "width": None, @@ -186,7 +322,7 @@ def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Im # 1. Try EXIF try: - if file_path.lower().endswith(('.jpg', '.jpeg', '.tiff', '.webp', '.png', '.heic', '.heif')): + if file_path.lower().endswith(IMAGE_EXTENSIONS): exif_dict = None img = None should_close = False @@ -209,13 +345,11 @@ def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Im metadata["exif_info"] = exif_dict - # Extract Date (DateTimeOriginal) - date_str = exif_dict.get("DateTimeOriginal") - if date_str: - try: - metadata["photo_time"] = datetime.strptime(date_str, '%Y:%m:%d %H:%M:%S') - except ValueError: - pass + # Extract Date + photo_time, source = extract_datetime_from_exif(exif_dict) + if photo_time: + metadata["photo_time"] = photo_time + metadata["photo_time_source"] = source # Extract GPS gps = get_gps_info(exif_dict) @@ -245,21 +379,45 @@ def extract_metadata(file_path: str, filename: str, image_obj: Optional[Image.Im print(traceback.format_exc()) print(f"Error extracting metadata: {e}") - # 2. If photo_time is still None, try Filename + # 2. If photo_time is still None, try video container metadata + if metadata["photo_time"] is None and file_path.lower().endswith(VIDEO_EXTENSIONS): + try: + photo_time = extract_datetime_from_video_metadata(file_path) + if photo_time: + metadata["photo_time"] = photo_time + metadata["photo_time_source"] = "video_metadata:creation_time" + except Exception: + pass + + # 3. If photo_time is still None, try Filename if metadata["photo_time"] is None: try: photo_time = extract_datetime_from_filename(filename) if photo_time: metadata["photo_time"] = photo_time - else: - photo_time = get_file_time_form_system(file_path) + metadata["photo_time_source"] = "filename" + except Exception: + pass + + # 4. If photo_time is still None, try dated folders + if metadata["photo_time"] is None: + try: + photo_time = extract_datetime_from_path(file_path) + if photo_time: metadata["photo_time"] = photo_time + metadata["photo_time_source"] = "path" except Exception: pass - # 3. Fallback to current time + # 5. Fallback to filesystem mtime + if metadata["photo_time"] is None: + photo_time = get_file_time_form_system(file_path) + metadata["photo_time"] = photo_time + metadata["photo_time_source"] = "file_mtime" + + # 6. Fallback to current time if metadata["photo_time"] is None: metadata["photo_time"] = datetime.now() + metadata["photo_time_source"] = "now" return metadata - diff --git a/package/server/app/utils/filename.py b/package/server/app/utils/filename.py index 0215cc36..5e8ce851 100644 --- a/package/server/app/utils/filename.py +++ b/package/server/app/utils/filename.py @@ -85,6 +85,8 @@ def _extract_datetime_from_filename(filename)-> datetime | None: r"(\d{8})[ _\.-](\d{2})[ _\.-](\d{2})[ _\.-](\d{2})", # 格式：YYYYMMDD_HH-MM-SS r"(\d{8})[ _T\.-](\d{6})", # 格式：YYYYMMDD_HHMMSS r"(\d{14})", # 格式：YYYYMMDDHHMMSS + r"(? datetime | None: return dt else: return is_valid_timestamp(timestamp, filename) + elif len(match.groups()) == 3: # 格式：YYYY-MM-DD / YYYYMMDD + year, month, day = match.groups() + dt = datetime(int(year), int(month), int(day)) + return dt elif len(match.groups()) == 4: # 格式：YYYY-MM-DD HHMMSS year, month, day, time_str = match.groups() if len(year) == 4: @@ -153,4 +159,4 @@ def extract_datetime_from_filename(filename) -> datetime | None: for filename in test_filenames: dt = extract_datetime_from_filename(filename) - print(f"Filename: {filename} -> Extracted Datetime: {dt}") \ No newline at end of file + print(f"Filename: {filename} -> Extracted Datetime: {dt}") diff --git a/package/server/tests/test_media_file.py b/package/server/tests/test_media_file.py new file mode 100644 index 00000000..f4a51aa2 --- /dev/null +++ b/package/server/tests/test_media_file.py @@ -0,0 +1,58 @@ +import asyncio +import unittest +import uuid +from unittest.mock import ANY, patch + +from starlette.responses import FileResponse + +from app.api import media + + +class _FakePhoto: + def __init__(self, photo_id): + self.id = photo_id + self.owner_id = uuid.uuid4() + self.file_path = "/app/SynologyPhotos/rowankid/IMG_0001.HEIC" + + +class _FakeQuery: + def __init__(self, photo): + self.photo = photo + + def filter(self, *_args, **_kwargs): + return self + + def first(self): + return self.photo + + +class _FakeDb: + def __init__(self, photo): + self.photo = photo + + def query(self, _model): + return _FakeQuery(self.photo) + + +class MediaFileTest(unittest.TestCase): + def test_heic_file_endpoint_serves_medium_preview(self): + photo_id = uuid.uuid4() + photo = _FakePhoto(photo_id) + db = _FakeDb(photo) + preview_path = "/app/data/thumbnails/aa/bb/preview.webp" + + with ( + patch.object(media, "_get_thumbnail_path", return_value=preview_path) as get_thumbnail_path, + patch.object(media.os.path, "exists", return_value=True), + patch.object(media.os.path, "getsize", return_value=123), + ): + response = asyncio.run(media.get_media_file(photo_id, request=None, range=None, db=db)) + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.path, preview_path) + self.assertEqual(response.media_type, "image/webp") + get_thumbnail_path.assert_called_once_with(photo.owner_id, photo_id, ANY, "medium") + + +if __name__ == "__main__": + unittest.main() diff --git a/package/server/tests/test_photo_time_extraction.py b/package/server/tests/test_photo_time_extraction.py new file mode 100644 index 00000000..88180c88 --- /dev/null +++ b/package/server/tests/test_photo_time_extraction.py @@ -0,0 +1,33 @@ +from datetime import datetime +import unittest + +from app.utils.exif import extract_datetime_from_path, parse_datetime_value +from app.utils.filename import extract_datetime_from_filename + + +class PhotoTimeExtractionTest(unittest.TestCase): + def test_extract_date_only_from_filename(self): + self.assertEqual(extract_datetime_from_filename("20250227").date(), datetime(2025, 2, 27).date()) + self.assertEqual(extract_datetime_from_filename("album 2024-3-28").date(), datetime(2024, 3, 28).date()) + + def test_extract_date_from_dated_folder(self): + path = "/app/Photos/20250227/DSC_1991.JPG" + + self.assertEqual(extract_datetime_from_path(path), datetime(2025, 2, 27)) + + def test_extract_inherited_year_month_day_from_folder(self): + path = "/app/Photos/2020.09.13罗文琪&文晶photo/9.15/jpg/1A5A8450z.jpg" + + self.assertEqual(extract_datetime_from_path(path), datetime(2020, 9, 15)) + + def test_extract_short_year_folder_date(self): + path = "/app/Photos/文晶写真照片/211004大唐芙蓉园（文女士）/DSC09505.JPG" + + self.assertEqual(extract_datetime_from_path(path), datetime(2021, 10, 4)) + + def test_parse_exif_datetime_fallback_value(self): + self.assertEqual(parse_datetime_value("2024:10:28 09:24:47"), datetime(2024, 10, 28, 9, 24, 47)) + + +if __name__ == "__main__": + unittest.main() diff --git a/package/server/tests/test_scan_filters.py b/package/server/tests/test_scan_filters.py new file mode 100644 index 00000000..2ef1d5df --- /dev/null +++ b/package/server/tests/test_scan_filters.py @@ -0,0 +1,24 @@ +import os +import tempfile +import unittest + +from app.service.tasks.scan import scan_directory_recursive + + +class ScanFiltersTest(unittest.TestCase): + def test_scan_skips_synology_eadir_metadata(self): + with tempfile.TemporaryDirectory() as tmp: + real_photo = os.path.join(tmp, "IMG_0001.JPG") + os.makedirs(os.path.join(tmp, "@eaDir", "IMG_0001.JPG")) + metadata_thumb = os.path.join(tmp, "@eaDir", "IMG_0001.JPG", "SYNOPHOTO_THUMB_M.jpg") + + open(real_photo, "wb").close() + open(metadata_thumb, "wb").close() + + found = scan_directory_recursive(tmp, {".jpg", ".jpeg"}) + + self.assertEqual(found, {real_photo}) + + +if __name__ == "__main__": + unittest.main() diff --git a/package/server/tests/test_task_worker_chunking.py b/package/server/tests/test_task_worker_chunking.py new file mode 100644 index 00000000..acb5f580 --- /dev/null +++ b/package/server/tests/test_task_worker_chunking.py @@ -0,0 +1,16 @@ +import unittest + +from app.db.models.task import TaskType +from app.service.task_worker import get_chunk_size + + +class TaskWorkerChunkingTest(unittest.TestCase): + def test_face_recognition_uses_single_image_batches(self): + self.assertEqual(get_chunk_size(TaskType.RECOGNIZE_FACE), 1) + + def test_classification_uses_small_batches(self): + self.assertEqual(get_chunk_size(TaskType.CLASSIFY_IMAGE), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/package/website/src/components/home/FaceSection.vue b/package/website/src/components/home/FaceSection.vue index 9ffa2807..30ffdbcc 100644 --- a/package/website/src/components/home/FaceSection.vue +++ b/package/website/src/components/home/FaceSection.vue @@ -5,7 +5,7 @@

人物相册

{{ data.total_identified }}位已识别 diff --git a/package/website/src/router/index.ts b/package/website/src/router/index.ts index 2134beef..9ea87326 100644 --- a/package/website/src/router/index.ts +++ b/package/website/src/router/index.ts @@ -62,6 +62,9 @@ const routes: RouteRecordRaw[] = [ ], }, + { path: '/people', redirect: '/album/people' }, + { path: '/people/:id', redirect: to => `/album/people/${to.params.id}` }, + // Annual Report (Standalone Layout) { path: '/annual-report', diff --git a/package/website/src/views/toolbox/CleanupPage.vue b/package/website/src/views/toolbox/CleanupPage.vue index 6e3eae21..8a11139f 100644 --- a/package/website/src/views/toolbox/CleanupPage.vue +++ b/package/website/src/views/toolbox/CleanupPage.vue @@ -114,6 +114,15 @@ const subtitle = computed(() => { return `${photos.value.length}${hasMore.value ? '+' : ''} 张照片` }) +const dedupePhotos = (items: AlbumImage[], existingIds = new Set()) => { + const seen = new Set(existingIds) + return items.filter(item => { + if (seen.has(item.id)) return false + seen.add(item.id) + return true + }) +} + const fetchPhotos = async (isLoadMore = false) => { if (loading.value) return @@ -132,18 +141,21 @@ const fetchPhotos = async (isLoadMore = false) => { }) const newPhotos = res.map(item => mapPhotoToImage(item)) + const uniqueNewPhotos = isLoadMore + ? dedupePhotos(newPhotos, new Set(photos.value.map(item => item.id))) + : dedupePhotos(newPhotos) if (newPhotos.length < limit) { hasMore.value = false } if (isLoadMore) { - photos.value.push(...newPhotos) + photos.value.push(...uniqueNewPhotos) } else { - photos.value = newPhotos + photos.value = uniqueNewPhotos } - skip.value += limit + skip.value += newPhotos.length } catch (e) { console.error('Fetch failed:', e) ElMessage.error('获取照片失败')