From de4c78240a9b1a9a96f5085a16a6a07c86b89d8c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 Jan 2026 00:34:47 +0000 Subject: [PATCH 1/5] Initial plan From 5fbdaaa7db9fc63f91573282a6eb734e68b3346c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 Jan 2026 00:38:06 +0000 Subject: [PATCH 2/5] Complete S3 storage implementation with express and recycleFiles methods Co-authored-by: SilianZ <113701655+SilianZ@users.noreply.github.com> --- core/storages/__init__.py | 13 +++++ core/storages/s3.py | 117 +++++++++++++++++++++++++++++++++++++- i18n/zh_cn.json | 6 ++ 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/core/storages/__init__.py b/core/storages/__init__.py index a9cfd8c..f6d3f3d 100644 --- a/core/storages/__init__.py +++ b/core/storages/__init__.py @@ -1,6 +1,7 @@ from core.classes import Storage from core.storages.local import LocalStorage from core.storages.alist import AListStorage +from core.storages.s3 import S3Storage from core.config import Config from typing import List @@ -20,4 +21,16 @@ def getStorages() -> List[Storage]: path=storage["path"], ) ) + if storage["type"] == "s3": + storages.append( + S3Storage( + endpoint=storage["endpoint"], + access_key_id=storage["access_key_id"], + secret_access_key=storage["secret_access_key"], + signature_version=storage.get("signature_version", "s3v4"), + bucket=storage["bucket"], + addressing_style=storage.get("addressing_style", "auto"), + session_token=storage.get("session_token", None), + ) + ) return storages diff --git a/core/storages/s3.py b/core/storages/s3.py index e7499e6..3b1890d 100644 --- a/core/storages/s3.py +++ b/core/storages/s3.py @@ -1,9 +1,12 @@ from core.classes import Storage, FileInfo, FileList from core.logger import logger +from core.i18n import locale from botocore.config import Config from botocore.exceptions import ClientError from typing import Literal, Union, Self from tqdm import tqdm +from aiohttp import web +from pathlib import Path import boto3 import humanize import io @@ -89,7 +92,8 @@ async def getMissingFiles(self, files: FileList, pbar: tqdm) -> FileList: s3_files = {} try: paginator = self.client.get_paginator("list_objects_v2") - async for page in paginator.paginate(Bucket=self.bucket): + # Use asyncio.to_thread to run synchronous paginator in thread pool + for page in paginator.paginate(Bucket=self.bucket): for obj in page.get("Contents", []): s3_files[obj["Key"]] = obj["Size"] except ClientError as e: @@ -100,4 +104,113 @@ async def getMissingFiles(self, files: FileList, pbar: tqdm) -> FileList: file_key = f"{file.hash[:2]}/{file.hash}" if file_key not in s3_files or s3_files[file_key] != file.size: missing_files.append(file) - pbar.update(1) \ No newline at end of file + pbar.update(1) + + return FileList(files=missing_files) + + async def express(self, hash: str, counter: dict) -> Union[web.Response, web.FileResponse]: + file_key = f"{hash[:2]}/{hash}" + try: + # Generate a presigned URL for the file + response = await asyncio.to_thread( + self.client.head_object, + Bucket=self.bucket, + Key=file_key + ) + file_size = response["ContentLength"] + + # Generate presigned URL for downloading the file + url = await asyncio.to_thread( + self.client.generate_presigned_url, + "get_object", + Params={"Bucket": self.bucket, "Key": file_key}, + ExpiresIn=3600 # URL valid for 1 hour + ) + + # Return redirect to presigned URL + response = web.HTTPFound(url) + response.headers["x-bmclapi-hash"] = hash + counter["bytes"] += file_size + counter["hits"] += 1 + return response + except ClientError as e: + if e.response["Error"]["Code"] == "404": + return web.HTTPNotFound() + logger.terror("storage.error.s3.express", hash=hash, e=e) + return web.HTTPError(text=str(e)) + except Exception as e: + logger.terror("storage.error.s3.express", hash=hash, e=e) + return web.HTTPError(text=str(e)) + + async def recycleFiles(self, files: FileList) -> None: + """Clean up files in S3 that are not in the valid file list""" + delete_files = [] + + # Get all valid file keys + valid_keys = { + f"{file.hash[:2]}/{file.hash}" + for file in files.files + } + + # List all files in S3 bucket + try: + paginator = self.client.get_paginator("list_objects_v2") + all_s3_keys = [] + for page in paginator.paginate(Bucket=self.bucket): + for obj in page.get("Contents", []): + all_s3_keys.append(obj["Key"]) + except ClientError as e: + logger.terror("storage.error.s3.recycle.list", e=e) + return + + # Find files to delete + with tqdm( + desc=locale.t("storage.tqdm.desc.recycling_check"), + total=len(all_s3_keys), + unit_scale=True, + unit=locale.t("storage.tqdm.unit.files"), + ) as pbar: + for key in all_s3_keys: + pbar.update(1) + if key not in valid_keys: + delete_files.append(key) + + if len(delete_files) == 0: + logger.tinfo("storage.success.s3.no_need_to_recycle") + return + + # Delete files in batches (S3 allows max 1000 objects per delete request) + with tqdm( + desc=locale.t("storage.tqdm.desc.recycling"), + total=len(delete_files), + unit_scale=True, + unit=locale.t("storage.tqdm.unit.files"), + ) as pbar: + total_size = 0 + for i in range(0, len(delete_files), 1000): + batch = delete_files[i:i+1000] + try: + # Get sizes before deleting + for key in batch: + try: + response = self.client.head_object(Bucket=self.bucket, Key=key) + total_size += response["ContentLength"] + except Exception: + pass + + # Delete batch + self.client.delete_objects( + Bucket=self.bucket, + Delete={ + "Objects": [{"Key": key} for key in batch], + "Quiet": True + } + ) + pbar.update(len(batch)) + except ClientError as e: + logger.terror("storage.error.s3.recycle", e=e) + + logger.tsuccess( + "storage.success.s3.recycled", + size=humanize.naturalsize(total_size, binary=True), + ) \ No newline at end of file diff --git a/i18n/zh_cn.json b/i18n/zh_cn.json index 3e03aec..353f26f 100644 --- a/i18n/zh_cn.json +++ b/i18n/zh_cn.json @@ -35,6 +35,12 @@ "storage.error.s3.write_file.size_mismatch": "无法校验 S3 储存文件 ${file} 的大小。理论值：${file_size}，实际值：${actual_file_size}。", "storage.error.s3.write_file.retry": "在尝试写入 S3 储存文件 ${file} 时遇到错误：${e}，将在 ${retry}s 后重试。", "storage.error.s3.write_file.failed": "无法写入 S3 储存文件 ${file}，已达到最高重试次数。", + "storage.error.s3.get_s3_files": "无法获取 S3 储存文件列表：${e}。", + "storage.error.s3.express": "无法从 S3 储存获取文件 ${hash}：${e}。", + "storage.error.s3.recycle.list": "无法列出 S3 储存文件进行回收：${e}。", + "storage.error.s3.recycle": "无法回收 S3 储存文件：${e}。", + "storage.success.s3.recycled": "成功回收了 ${size} 的 S3 储存文件。", + "storage.success.s3.no_need_to_recycle": "当前 S3 储存无需要回收的文件。", "i18n.prompt.failed": "（i18n 字符串解析失败）", "cluster.info.filelist.fetching": "正在获取文件列表……", "cluster.success.filelist.fetched": "成功获取文件列表！", From 5f0afb5880a28ba4aebf43487d2bb5eab1747a94 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 Jan 2026 00:39:21 +0000 Subject: [PATCH 3/5] Improve dashboard with refresh button, better polling, error handling and updated README Co-authored-by: SilianZ <113701655+SilianZ@users.noreply.github.com> --- dashboard/README.md | 74 ++++++++++++++++++++++++++++++-- dashboard/src/App.vue | 2 +- dashboard/src/views/HomeView.vue | 40 ++++++++++++++++- 3 files changed, 110 insertions(+), 6 deletions(-) diff --git a/dashboard/README.md b/dashboard/README.md index 33895ab..7347757 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -1,5 +1,73 @@ -# Vue 3 + TypeScript + Vite +# Python-OpenBMCLAPI Dashboard -This template should help get you started developing with Vue 3 and TypeScript in Vite. The template uses Vue 3 ` @@ -41,9 +59,27 @@ h1 { unicode-bidi: isolate; } +.refresh-btn { + border: 1px solid var(--p-content-border-color); + background: var(--p-content-background); + cursor: pointer; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.error-message { + display: flex; + align-items: center; +} + @media screen and (max-width: 720px) { h1 { font-size: 30px; } + .refresh-btn { + font-size: 14px; + padding: 0.5rem 1rem; + } } From 032a52a7b5b5c8b2f3ef1240ded7ecf57c90016e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 Jan 2026 00:41:26 +0000 Subject: [PATCH 4/5] Restructure and enhance documentation with detailed setup and configuration guides Co-authored-by: SilianZ <113701655+SilianZ@users.noreply.github.com> --- README.md | 200 +++++++++++++++++++++++++++++++++++++++++-- i18n/README_en.md | 214 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 393 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 554b0f0..50a125c 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,199 @@ -# 简介 +## 目录 + +- [简介](#简介) +- [核心特性](#核心特性) +- [快速开始](#快速开始) + - [前置要求](#前置要求) + - [安装步骤](#安装步骤) + - [Docker 部署](#docker-部署) +- [配置说明](#配置说明) + - [基础配置](#基础配置) + - [存储配置](#存储配置) +- [贡献](#贡献) +- [鸣谢](#鸣谢) + +## 简介 本项目是 [OpenBMCLAPI](https://github.com/bangbang93/openbmclapi) 的 Python 版本，OpenBMCLAPI 是通过分布式集群帮助 [BMCLAPI](https://bmclapidoc.bangbang93.com/) 进行文件分发、加速中国大陆 Minecraft 下载的公益项目。 如果你想加入 OpenBMCLAPI，可以寻找 [bangbang93](https://github.com/bangbang93) 获取 `CLUSTER_ID` 和 `CLUSTER_SECRET`。 -# 贡献 +## 核心特性 + +### 🎯 多存储后端支持 + +- **本地存储** - 传统的本地文件系统存储 +- **AList** - 支持通过 AList 进行文件管理和分发 +- **S3 兼容存储** - 支持 AWS S3、MinIO、阿里云 OSS 等 S3 兼容的对象存储服务 + +### 📊 实时监控仪表盘 + +- 实时性能监控（CPU、内存、连接数） +- 多维度请求统计（小时/天/月） +- 用户分布可视化 +- 响应式设计，支持移动设备访问 + +### 🚀 高性能架构 + +- 异步 I/O 处理，提高并发性能 +- 智能文件同步机制 +- 自动回收过期文件 +- 支持断点续传 + +### 🌍 国际化支持 + +- 多语言界面支持 +- 通过 Crowdin 协作翻译 + +## 快速开始 + +### 前置要求 + +- Python 3.12 或更高版本 +- pip 包管理器 +- （可选）Docker 和 Docker Compose + +### 安装步骤 + +1. **克隆仓库** + +```bash +git clone https://github.com/TTB-Network/python-openbmclapi-v2.git +cd python-openbmclapi-v2 +``` + +2. **安装依赖** + +```bash +pip install -r requirements.txt +# 或使用 Poetry +poetry install +``` + +3. **配置应用** + +编辑 `config/config.yml` 文件，填入你的集群配置： + +```yaml +cluster: + id: "你的集群ID" + secret: "你的集群密钥" + host: "你的域名或IP" + port: 8800 +``` + +4. **启动应用** + +```bash +python main.py +``` + +### Docker 部署 + +使用 Docker 是最简单的部署方式： + +```bash +docker run -d \ + --name python-openbmclapi \ + -p 8800:8800 \ + -v /path/to/cache:/app/cache \ + -v /path/to/config:/app/config \ + silianz/python-openbmclapi-v2:latest +``` + +或使用 Docker Compose： + +```yaml +version: '3' +services: + openbmclapi: + image: silianz/python-openbmclapi-v2:latest + ports: + - "8800:8800" + volumes: + - ./cache:/app/cache + - ./config:/app/config + restart: unless-stopped +``` + +## 配置说明 + +### 基础配置 + +配置文件位于 `config/config.yml`： + +```yaml +cluster: + base_url: "https://openbmclapi.bangbang93.com" # 主控地址 + id: "" # 集群 ID + secret: "" # 集群密钥 + host: "" # 公网访问地址 + byoc: false # 是否使用自己的证书 + public_port: -1 # 公网端口（-1 表示使用 port） + port: 8800 # 监听端口 + +advanced: + lang: "zh_cn" # 语言设置 + debug: false # 调试模式 + retry: 5 # 重试次数 + delay: 15 # 重试延迟（秒） + keep_alive: 60 # 保活间隔（秒） + sync_interval: 120 # 同步间隔（秒） +``` + +### 存储配置 + +支持配置多个存储后端，系统会依次尝试从各存储读取文件： + +#### 本地存储 + +```yaml +storages: + - type: "local" + path: "./cache" +``` + +#### AList 存储 + +```yaml +storages: + - type: "alist" + url: "http://your-alist-server.com" + username: "admin" + password: "password" + path: "/openbmclapi/" +``` + +#### S3 兼容存储 + +```yaml +storages: + - type: "s3" + endpoint: "https://s3.amazonaws.com" # S3 端点地址 + access_key_id: "your-access-key" # 访问密钥 ID + secret_access_key: "your-secret-key" # 访问密钥 + bucket: "your-bucket-name" # 存储桶名称 + signature_version: "s3v4" # 签名版本（可选） + addressing_style: "auto" # 寻址样式：auto/path/virtual（可选） + session_token: null # 会话令牌（可选） +``` + +**S3 配置说明：** + +- `endpoint`: S3 服务端点 + - AWS S3: `https://s3.amazonaws.com` 或区域端点如 `https://s3.us-west-2.amazonaws.com` + - MinIO: `http://your-minio-server:9000` + - 阿里云 OSS: `https://oss-cn-hangzhou.aliyuncs.com` +- `signature_version`: 签名版本，通常使用 `s3v4` +- `addressing_style`: + - `auto`: 自动选择 + - `path`: 路径样式（`endpoint/bucket/key`） + - `virtual`: 虚拟主机样式（`bucket.endpoint/key`） + +## 贡献 如果你有能力，你可以向我们的仓库提交 Pull Request 或 Issue。 @@ -49,10 +235,10 @@ 在贡献之前，请先阅读我们的[贡献准则](./CONTRIBUTING.md)。 -# 鸣谢 - -[LiterMC/go-openbmclapi](https://github.com/LiterMC/go-openbmclapi) +## 鸣谢 -[bangbang93/openbmclapi](https://github.com/bangbang93/openbmclapi) +感谢以下项目为本项目提供的灵感和参考： -[SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI) +- [LiterMC/go-openbmclapi](https://github.com/LiterMC/go-openbmclapi) +- [bangbang93/openbmclapi](https://github.com/bangbang93/openbmclapi) +- [SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI) diff --git a/i18n/README_en.md b/i18n/README_en.md index 3fc39d5..0479887 100644 --- a/i18n/README_en.md +++ b/i18n/README_en.md @@ -6,9 +6,9 @@ -# OpenBMCLAPI for Python +# OpenBMCLAPI for Python v2 -简体中文 | [English](/i18n/README_en.md) +[简体中文](/README.md) | English ![GitHub Issues or Pull Requests](https://img.shields.io/github/issues-pr/TTB-Network/python-openbmclapi) ![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/TTB-Network/python-openbmclapi) @@ -25,21 +25,207 @@ ✨ Python implementation based on [OpenBMCLAPI](https://github.com/bangbang93/openbmclapi). -🎨 **Cross system** and **cross framework**.This is thanks to the powerful language features of Python. +🎨 **Cross-platform** and **cross-architecture**. Thanks to the powerful features of Python. -✨ **Docker** support.With the help of Docker, you can deploy python-openbmclapi in a **short time**. +✨ **Docker** support. Deploy python-openbmclapi quickly with Docker. -🎉 **\* New features!\*** WebDAV support.Through WebDAV (Web-based Distorting and Versioning), users can edit and manage files stored on web servers in collaboration with them. +🎉 **New Feature!** Dashboard. Built with Vue.js for a modern, clean interface. -# Introduction +## Table of Contents -This project is a Python version of [OpenBMCLAPI](https://github.com/bangbang93/openbmclapi). OpenBMCLAPI is a file distribution cluster to help [BMCLAPI](https://bmclapidoc.bangbang93.com/) speed up the file download of Minecraft in the Chinese mainland. +- [Introduction](#introduction) +- [Core Features](#core-features) +- [Quick Start](#quick-start) + - [Prerequisites](#prerequisites) + - [Installation](#installation) + - [Docker Deployment](#docker-deployment) +- [Configuration](#configuration) + - [Basic Configuration](#basic-configuration) + - [Storage Configuration](#storage-configuration) +- [Contributing](#contributing) +- [Acknowledgments](#acknowledgments) -If you want to join OpenBMCLAPI, you can ask [bangbang93](https://github.com/bangbang93) for an `CLUSTER_ID` and an `CLUSTER_SECRET`. +## Introduction -# Contributing +This project is a Python version of [OpenBMCLAPI](https://github.com/bangbang93/openbmclapi). OpenBMCLAPI is a file distribution cluster that helps [BMCLAPI](https://bmclapidoc.bangbang93.com/) accelerate Minecraft downloads in mainland China. + +If you want to join OpenBMCLAPI, contact [bangbang93](https://github.com/bangbang93) to obtain a `CLUSTER_ID` and `CLUSTER_SECRET`. + +## Core Features + +### 🎯 Multiple Storage Backend Support + +- **Local Storage** - Traditional local filesystem storage +- **AList** - File management and distribution through AList +- **S3-Compatible Storage** - Support for AWS S3, MinIO, Alibaba Cloud OSS, and other S3-compatible object storage services + +### 📊 Real-time Monitoring Dashboard + +- Real-time performance monitoring (CPU, memory, connections) +- Multi-dimensional request statistics (hourly/daily/monthly) +- User distribution visualization +- Responsive design with mobile device support + +### 🚀 High-Performance Architecture + +- Asynchronous I/O processing for better concurrency +- Smart file synchronization mechanism +- Automatic cleanup of expired files +- Resume download support + +### 🌍 Internationalization Support + +- Multi-language interface +- Collaborative translation via Crowdin + +## Quick Start + +### Prerequisites + +- Python 3.12 or higher +- pip package manager +- (Optional) Docker and Docker Compose + +### Installation + +1. **Clone the repository** + +```bash +git clone https://github.com/TTB-Network/python-openbmclapi-v2.git +cd python-openbmclapi-v2 +``` + +2. **Install dependencies** + +```bash +pip install -r requirements.txt +# Or using Poetry +poetry install +``` + +3. **Configure the application** + +Edit `config/config.yml` and fill in your cluster configuration: + +```yaml +cluster: + id: "your-cluster-id" + secret: "your-cluster-secret" + host: "your-domain-or-ip" + port: 8800 +``` + +4. **Start the application** + +```bash +python main.py +``` + +### Docker Deployment + +Using Docker is the easiest way to deploy: + +```bash +docker run -d \ + --name python-openbmclapi \ + -p 8800:8800 \ + -v /path/to/cache:/app/cache \ + -v /path/to/config:/app/config \ + silianz/python-openbmclapi-v2:latest +``` + +Or use Docker Compose: + +```yaml +version: '3' +services: + openbmclapi: + image: silianz/python-openbmclapi-v2:latest + ports: + - "8800:8800" + volumes: + - ./cache:/app/cache + - ./config:/app/config + restart: unless-stopped +``` + +## Configuration + +### Basic Configuration + +Configuration file located at `config/config.yml`: + +```yaml +cluster: + base_url: "https://openbmclapi.bangbang93.com" # Master server URL + id: "" # Cluster ID + secret: "" # Cluster secret + host: "" # Public access address + byoc: false # Bring your own certificate + public_port: -1 # Public port (-1 uses port value) + port: 8800 # Listening port + +advanced: + lang: "zh_cn" # Language setting + debug: false # Debug mode + retry: 5 # Retry count + delay: 15 # Retry delay (seconds) + keep_alive: 60 # Keep-alive interval (seconds) + sync_interval: 120 # Sync interval (seconds) +``` + +### Storage Configuration + +Support multiple storage backends. The system will try to read files from each storage in sequence: + +#### Local Storage + +```yaml +storages: + - type: "local" + path: "./cache" +``` + +#### AList Storage + +```yaml +storages: + - type: "alist" + url: "http://your-alist-server.com" + username: "admin" + password: "password" + path: "/openbmclapi/" +``` + +#### S3-Compatible Storage + +```yaml +storages: + - type: "s3" + endpoint: "https://s3.amazonaws.com" # S3 endpoint URL + access_key_id: "your-access-key" # Access key ID + secret_access_key: "your-secret-key" # Secret access key + bucket: "your-bucket-name" # Bucket name + signature_version: "s3v4" # Signature version (optional) + addressing_style: "auto" # Addressing style: auto/path/virtual (optional) + session_token: null # Session token (optional) +``` + +**S3 Configuration Details:** + +- `endpoint`: S3 service endpoint + - AWS S3: `https://s3.amazonaws.com` or regional endpoint like `https://s3.us-west-2.amazonaws.com` + - MinIO: `http://your-minio-server:9000` + - Alibaba Cloud OSS: `https://oss-cn-hangzhou.aliyuncs.com` +- `signature_version`: Signature version, usually `s3v4` +- `addressing_style`: + - `auto`: Automatic selection + - `path`: Path-style (`endpoint/bucket/key`) + - `virtual`: Virtual-hosted-style (`bucket.endpoint/key`) + +## Contributing If you have the ability, you can submit a Pull Request or Issue to our repository. @@ -47,10 +233,10 @@ If you want to help us with multilingual translations, go to [Crowdin](https://t Please read our [Contributing Rules](./CONTRIBUTING_en.md) before contributing. -# Thanks - -[LiterMC/go-openbmclapi](https://github.com/LiterMC/go-openbmclapi) +## Acknowledgments -[bangbang93/openbmclapi](https://github.com/bangbang93/openbmclapi) +Thanks to the following projects for inspiration and reference: -[SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI) +- [LiterMC/go-openbmclapi](https://github.com/LiterMC/go-openbmclapi) +- [bangbang93/openbmclapi](https://github.com/bangbang93/openbmclapi) +- [SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI) From 3ae64b4ec4c00962044655361a9af82fc67a7470 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 Jan 2026 00:43:30 +0000 Subject: [PATCH 5/5] Fix async/await issues in S3 storage and adjust dashboard polling interval Co-authored-by: SilianZ <113701655+SilianZ@users.noreply.github.com> --- core/storages/s3.py | 96 +++++++++++++++++++------------- dashboard/src/views/HomeView.vue | 2 +- 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/core/storages/s3.py b/core/storages/s3.py index 3b1890d..6089d48 100644 --- a/core/storages/s3.py +++ b/core/storages/s3.py @@ -91,11 +91,16 @@ async def check(self) -> None: async def getMissingFiles(self, files: FileList, pbar: tqdm) -> FileList: s3_files = {} try: - paginator = self.client.get_paginator("list_objects_v2") - # Use asyncio.to_thread to run synchronous paginator in thread pool - for page in paginator.paginate(Bucket=self.bucket): - for obj in page.get("Contents", []): - s3_files[obj["Key"]] = obj["Size"] + # Run synchronous paginator in thread pool to avoid blocking event loop + def list_s3_files(): + files_dict = {} + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket): + for obj in page.get("Contents", []): + files_dict[obj["Key"]] = obj["Size"] + return files_dict + + s3_files = await asyncio.to_thread(list_s3_files) except ClientError as e: logger.terror("storage.error.s3.get_s3_files", e=e) @@ -111,21 +116,21 @@ async def getMissingFiles(self, files: FileList, pbar: tqdm) -> FileList: async def express(self, hash: str, counter: dict) -> Union[web.Response, web.FileResponse]: file_key = f"{hash[:2]}/{hash}" try: - # Generate a presigned URL for the file - response = await asyncio.to_thread( - self.client.head_object, - Bucket=self.bucket, - Key=file_key - ) - file_size = response["ContentLength"] + # Run S3 operations in thread pool to avoid blocking event loop + def get_presigned_url(): + # Check if file exists and get size + response = self.client.head_object(Bucket=self.bucket, Key=file_key) + file_size = response["ContentLength"] + + # Generate presigned URL for downloading the file + url = self.client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": file_key}, + ExpiresIn=3600 # URL valid for 1 hour + ) + return url, file_size - # Generate presigned URL for downloading the file - url = await asyncio.to_thread( - self.client.generate_presigned_url, - "get_object", - Params={"Bucket": self.bucket, "Key": file_key}, - ExpiresIn=3600 # URL valid for 1 hour - ) + url, file_size = await asyncio.to_thread(get_presigned_url) # Return redirect to presigned URL response = web.HTTPFound(url) @@ -152,13 +157,17 @@ async def recycleFiles(self, files: FileList) -> None: for file in files.files } - # List all files in S3 bucket + # List all files in S3 bucket - run in thread pool to avoid blocking try: - paginator = self.client.get_paginator("list_objects_v2") - all_s3_keys = [] - for page in paginator.paginate(Bucket=self.bucket): - for obj in page.get("Contents", []): - all_s3_keys.append(obj["Key"]) + def list_all_s3_keys(): + keys = [] + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket): + for obj in page.get("Contents", []): + keys.append(obj["Key"]) + return keys + + all_s3_keys = await asyncio.to_thread(list_all_s3_keys) except ClientError as e: logger.terror("storage.error.s3.recycle.list", e=e) return @@ -190,22 +199,29 @@ async def recycleFiles(self, files: FileList) -> None: for i in range(0, len(delete_files), 1000): batch = delete_files[i:i+1000] try: - # Get sizes before deleting - for key in batch: - try: - response = self.client.head_object(Bucket=self.bucket, Key=key) - total_size += response["ContentLength"] - except Exception: - pass + # Run batch operations in thread pool + def process_batch(): + size = 0 + # Get sizes before deleting + for key in batch: + try: + response = self.client.head_object(Bucket=self.bucket, Key=key) + size += response["ContentLength"] + except Exception: + pass + + # Delete batch + self.client.delete_objects( + Bucket=self.bucket, + Delete={ + "Objects": [{"Key": key} for key in batch], + "Quiet": True + } + ) + return size - # Delete batch - self.client.delete_objects( - Bucket=self.bucket, - Delete={ - "Objects": [{"Key": key} for key in batch], - "Quiet": True - } - ) + batch_size = await asyncio.to_thread(process_batch) + total_size += batch_size pbar.update(len(batch)) except ClientError as e: logger.terror("storage.error.s3.recycle", e=e) diff --git a/dashboard/src/views/HomeView.vue b/dashboard/src/views/HomeView.vue index d1e6a63..de4d817 100644 --- a/dashboard/src/views/HomeView.vue +++ b/dashboard/src/views/HomeView.vue @@ -8,7 +8,7 @@ import { ref } from 'vue' import { watch } from 'vue' const { data, refresh, loading, error } = useRequest((): Promise => fetchStat(), { - pollingInterval: 5000, // Poll every 5 seconds for real-time updates + pollingInterval: 10000, // Poll every 10 seconds for real-time updates without excessive server load refreshOnWindowFocus: true // Refresh when user returns to tab })