diff --git a/kbmanager/app_logic.py b/kbmanager/app_logic.py index 6f30a1a..57cdc9c 100644 --- a/kbmanager/app_logic.py +++ b/kbmanager/app_logic.py @@ -99,7 +99,7 @@ async def upload_file_to_kb(self, file_path_str: str, kb_id: str): click.echo(f"Error: {e}", err=True) sys.exit(1) - async def upload_directory_to_kb(self, directory_path_str: str, kb_id: str, ignore_file_path: Optional[str] = None): + async def upload_directory_to_kb(self, directory_path_str: str, kb_id: str, ignore_file_path: Optional[str] = None, skip_existing: Optional[bool] = None): directory_path = Path(directory_path_str) if not directory_path.is_dir(): raise FileOperationError(f"Directory not found: {directory_path_str}") @@ -118,12 +118,29 @@ async def upload_directory_to_kb(self, directory_path_str: str, kb_id: str, igno files_to_upload: List[Tuple[Path, str]] = [] + if skip_existing: + existing_files = [ + f.meta.additional_properties["name"] + for f in await self._api.list_files_for_knowledge_base(kb_id) + if ( + hasattr(f, "meta") + and isinstance(f.meta, self._api.FileMetadataResponseMeta) + and "name" in f.meta.additional_properties + ) + ] + else: + existing_files = [] + for root, _, files_in_dir in os.walk(directory_path): current_dir_path = Path(root) for file_name in files_in_dir: # Renamed 'files' to 'files_in_dir' to avoid conflict with outer 'files_to_upload' full_file_path = current_dir_path / file_name relative_file_path = str(full_file_path.relative_to(directory_path)) + if file_name in existing_files: + logging.debug(f"Skipping existing file: {relative_file_path}") + continue + if not kbignore_parser.is_ignored(str(relative_file_path)): files_to_upload.append((full_file_path, relative_file_path)) else: diff --git a/kbmanager/main.py b/kbmanager/main.py index ab42754..9423202 100644 --- a/kbmanager/main.py +++ b/kbmanager/main.py @@ -184,16 +184,22 @@ async def upload_file(ctx: CLIContext, file_path, kb_id): type=click.Path(exists=True, dir_okay=False, resolve_path=True), help="Path to a .kbignore file. Defaults to .kbignore in the directory_path if not specified.", ) +@click.option( + "--skip-existing", + 'skip_existing', + flag_value=True, + help="Skip existing files in knowledge base. Files are matched on file name.", +) @click.pass_obj @coro -async def upload_dir(ctx: CLIContext, directory_path, kb_id, ignore_file): +async def upload_dir(ctx: CLIContext, directory_path, kb_id, ignore_file, skip_existing): """ Uploads files from a directory to a specified knowledge base, respecting .kbignore. """ ctx.init_api_client_and_app_logic() click.echo(f"Uploading directory: {directory_path}") try: - await ctx.app_logic.upload_directory_to_kb(directory_path, kb_id, ignore_file) + await ctx.app_logic.upload_directory_to_kb(directory_path, kb_id, ignore_file, skip_existing) except (APIError, FileOperationError) as e: logger.error(f"Could not complete directory upload: {e}") click.echo(f"Error: {e}", err=True)