From 333c2a4f4c56487f07f0bd1b6cc3163552560b62 Mon Sep 17 00:00:00 2001 From: RY150150 <87613708+RY150150@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:32:37 +0800 Subject: [PATCH 1/2] Harden PyTorch migration scripts and package layout --- README_BA_FPCC.md | 63 +++++++++++ datasets/__init__.py | 1 + datasets/fpcc_dataset.py | 54 +++++++++ docs/fpcc_simple_analysis.md | 52 +++++++++ losses/__init__.py | 1 + losses/boundary_loss.py | 40 +++++++ models/__init__.py | 1 + models/boundary_branch.py | 43 +++++++ models/fpcc_pytorch.py | 72 ++++++++++++ scripts/convert_syn_pbox_one_frame.py | 105 +++++++++++++++++ scripts/test_ba_fpcc.py | 144 ++++++++++++++++++++++++ scripts/train_ba_fpcc.py | 91 +++++++++++++++ scripts/visualize_syn_pbox_one_frame.py | 27 +++++ utils/boundary_utils.py | 57 ++++++++++ utils/obb_utils.py | 48 ++++++++ utils/simple_metrics.py | 64 +++++++++++ utils/simple_visualize.py | 52 +++++++++ 17 files changed, 915 insertions(+) create mode 100644 README_BA_FPCC.md create mode 100644 datasets/__init__.py create mode 100644 datasets/fpcc_dataset.py create mode 100644 docs/fpcc_simple_analysis.md create mode 100644 losses/__init__.py create mode 100644 losses/boundary_loss.py create mode 100644 models/__init__.py create mode 100644 models/boundary_branch.py create mode 100644 models/fpcc_pytorch.py create mode 100644 scripts/convert_syn_pbox_one_frame.py create mode 100644 scripts/test_ba_fpcc.py create mode 100644 scripts/train_ba_fpcc.py create mode 100644 scripts/visualize_syn_pbox_one_frame.py create mode 100644 utils/boundary_utils.py create mode 100644 utils/obb_utils.py create mode 100644 utils/simple_metrics.py create mode 100644 utils/simple_visualize.py diff --git a/README_BA_FPCC.md b/README_BA_FPCC.md new file mode 100644 index 0000000..46afea9 --- /dev/null +++ b/README_BA_FPCC.md @@ -0,0 +1,63 @@ +# BA-FPCC-OBB + +本版本提供 **PyTorch 主干迁移版**(不改坏原 FPCC TensorFlow 代码)。 + +## 安全建议(强烈推荐) +先在新分支开发,避免影响原始 FPCC: +```bash +git checkout -b feature/ba-fpcc-pytorch-migration +``` + +## 1) 原始 FPCC baseline(保留不变) +```bash +python fpcc_train.py --gpu 0 --input_list datas/ring_train.txt +python fpcc_test.py --gpu 0 --restore_dir checkpoint/ +``` + +## 2) PyTorch 版 BA-FPCC 训练 +```bash +python scripts/train_ba_fpcc.py \ + --input_list datas/ring_train.txt \ + --batch_size 4 \ + --epochs 100 \ + --use_boundary_loss \ + --use_boundary_branch \ + --lambda_boundary 1.0 \ + --lambda_bce 0.5 \ + --save_path checkpoint/ba_fpcc_torch.pt +``` + +## 3) PyTorch 版 BA-FPCC 测试 + OBB +```bash +python scripts/test_ba_fpcc.py \ + --ckpt checkpoint/ba_fpcc_torch.pt \ + --test_glob 'datas/ring_test/*.txt' \ + --use_boundary_branch \ + --use_obb --save_obb \ + --obb_json test_results/ba_fpcc_obb.json +``` + +## 4) SYN-PBOX 单帧转换 +```bash +python scripts/convert_syn_pbox_one_frame.py \ + --scene_camera path/to/scene_camera.json \ + --depth path/to/depth.png \ + --mask_dir path/to/mask_visib \ + --frame_id 0 \ + --output demo_syn_frame.npz +``` + +## 5) SYN-PBOX 单帧可视化 +```bash +python scripts/visualize_syn_pbox_one_frame.py --npz demo_syn_frame.npz --show_obb +``` + +## 说明 +- 原始 TF 训练/测试入口未删除,便于回归对照。 +- 新增 PyTorch 训练/测试脚本已使用原 FPCC 数据格式(h5/txt)并包含 BA 开关。 + + +## 环境依赖 +```bash +pip install numpy torch scikit-learn open3d imageio h5py scipy +``` diff --git a/datasets/__init__.py b/datasets/__init__.py new file mode 100644 index 0000000..e5521bf --- /dev/null +++ b/datasets/__init__.py @@ -0,0 +1 @@ +"""Dataset package for BA-FPCC-OBB.""" diff --git a/datasets/fpcc_dataset.py b/datasets/fpcc_dataset.py new file mode 100644 index 0000000..72f1df8 --- /dev/null +++ b/datasets/fpcc_dataset.py @@ -0,0 +1,54 @@ +"""PyTorch dataset wrappers for original FPCC h5 files.""" + +import numpy as np +import torch +from torch.utils.data import Dataset + +import provider + + +class FPCCDataset(Dataset): + def __init__(self, file_list_txt, point_dim=6, num_groups=50): + self.files = provider.getDataFiles(file_list_txt) + self.point_dim = point_dim + self.num_groups = num_groups + + self.data = [] + self.group = [] + self.center = [] + for f in self.files: + cur_data, cur_group, _, _, cur_score = provider.loadDataFile_with_groupseglabel_stanfordindoor(f) + self.data.append(cur_data) + self.group.append(cur_group) + self.center.append(cur_score) + + self.data = np.concatenate(self.data, axis=0).astype(np.float32) + self.group = np.concatenate(self.group, axis=0).astype(np.int64) + self.center = np.concatenate(self.center, axis=0).astype(np.float32) + + def __len__(self): + return self.data.shape[0] + + def __getitem__(self, idx): + points = self.data[idx, :, : self.point_dim] + instance_labels = np.asarray(self.group[idx]).reshape(-1) + center_labels = np.asarray(self.center[idx]).reshape(-1) + one_hot = group_to_one_hot(instance_labels, self.num_groups) + return { + "points": torch.from_numpy(points), + "instance_labels": torch.from_numpy(instance_labels), + "center_labels": torch.from_numpy(center_labels), + "group_one_hot": torch.from_numpy(one_hot), + } + + +def group_to_one_hot(group_labels, num_groups=50): + """group_labels: [N], output [N,num_groups].""" + group_labels = np.asarray(group_labels) + one_hot = np.zeros((group_labels.shape[0], num_groups), dtype=np.float32) + uniq = [u for u in np.unique(group_labels) if u >= 0] + mapping = {u: i for i, u in enumerate(uniq[:num_groups])} + for i, gid in enumerate(group_labels): + if gid in mapping: + one_hot[i, mapping[gid]] = 1.0 + return one_hot diff --git a/docs/fpcc_simple_analysis.md b/docs/fpcc_simple_analysis.md new file mode 100644 index 0000000..d1d9e38 --- /dev/null +++ b/docs/fpcc_simple_analysis.md @@ -0,0 +1,52 @@ +# FPCC 简要结构分析(第一步,仅阅读) + +> 说明:原始项目是 **TensorFlow 1.x** 实现,不是 PyTorch。若要在 BA-FPCC-OBB 中统一到 PyTorch,需要新增脚本做兼容/迁移。 + +1. **训练入口文件** + - `fpcc_train.py`。 + +2. **测试入口文件** + - `fpcc_test.py`。 + +3. **模型 forward 输入/输出** + - 输入来自 `models/model.py::get_model(backbone, point_cloud, is_training, ...)`。 + - `point_cloud` shape 约为 `[B, N, POINT_DIM]`(训练中一般取前 6 维)。 + - 输出是 dict: + - `center_score`: 每点中心置信度,shape 约 `[B, N]` + - `point_features`: 每点特征(Fsim),shape 约 `[B, N, 128]` + - `simmat`: 相似度矩阵,shape `[B, N, N]`(train=True 时) + - `3d_distance`: 点间 3D 距离矩阵,shape `[B, N, N]` + +4. **dataloader 返回字段** + - `provider.loadDataFile_with_groupseglabel_stanfordindoor(...)` 在训练里被调用,返回: + - `cur_data` + - `cur_group` + - `_`(未使用) + - `_`(未使用) + - `cur_score` + +5. **每个 batch 点云 shape** + - 训练 feed 到模型的是 `input_data[..., :POINT_DIM]`,shape `[B, 4096, POINT_DIM]`。 + +6. **instance label 与 center label shape** + - instance label(group label)在 one-hot 后为 `ptsgroup_label_ph`: `[B, N, NUM_GROUPS]`。 + - center label 为 `pts_score_ph`: `[B, N]`。 + +7. **loss 在哪里计算** + - 在 `models/model.py::get_loss(...)` 计算。 + - 在 `fpcc_train.py` 中通过 `loss, score_loss, grouperr = model.get_loss(...)` 调用。 + +8. **聚类在哪里做** + - 在 `fpcc_test.py` 中调用 `utils.test_utils` 里的 `GroupMerging_fpcc(...)` 完成实例聚类。 + +9. **加 boundary loss 建议改哪里** + - 最小侵入:新增训练脚本 `scripts/train_ba_fpcc.py`,在原始 loss 基础上额外做逐点加权。 + - 逐点 loss 可以优先作用在 center 回归项(`ptscenter_loss`)或新增点级监督项。 + +10. **加 boundary branch 建议改哪里** + - 最小侵入:不要改原 `fpcc_train.py`,在新脚本里拿 `net_output['point_features']` 后接一个轻量 BoundaryBranch。 + - 若 TF 图里插分支过于复杂,可先在 PyTorch 训练流程里演示并保留 TODO。 + +11. **加 OBB 后处理从哪里拿预测实例点云** + - 在 `fpcc_test.py` 聚类后得到 `ins_pre`(每点实例 id)。 + - 用 `pts[:,0:3]`(或对齐后的 `pts[:,3:6]`)按实例 id 取点,再做 OBB。 diff --git a/losses/__init__.py b/losses/__init__.py new file mode 100644 index 0000000..c3a6a8e --- /dev/null +++ b/losses/__init__.py @@ -0,0 +1 @@ +"""Loss package for BA-FPCC-OBB.""" diff --git a/losses/boundary_loss.py b/losses/boundary_loss.py new file mode 100644 index 0000000..c5db1a3 --- /dev/null +++ b/losses/boundary_loss.py @@ -0,0 +1,40 @@ +"""Simple boundary-related loss functions for BA-FPCC-OBB.""" + +import torch +import torch.nn.functional as F + + +def boundary_weighted_loss(point_loss, boundary_labels, lambda_boundary=1.0): + """Weighted mean point loss. + + point_loss: [B,N] or [N] + boundary_labels: same shape, {0,1} + """ + if point_loss.shape != boundary_labels.shape: + raise ValueError(f"shape mismatch: point_loss {point_loss.shape} vs boundary_labels {boundary_labels.shape}") + weights = 1.0 + lambda_boundary * boundary_labels.float() + return (point_loss * weights).mean() + + +def boundary_bce_loss(boundary_logits, boundary_labels): + """BCEWithLogits loss for boundary branch. + + boundary_logits: [B,N] or [N] + boundary_labels: same shape + """ + if boundary_logits.shape != boundary_labels.shape: + raise ValueError(f"shape mismatch: boundary_logits {boundary_logits.shape} vs boundary_labels {boundary_labels.shape}") + return F.binary_cross_entropy_with_logits(boundary_logits, boundary_labels.float()) + + +if __name__ == "__main__": + pl = torch.rand(2, 16) + bl = torch.randint(0, 2, (2, 16)) + lw = boundary_weighted_loss(pl, bl, lambda_boundary=1.0) + lw0 = boundary_weighted_loss(pl, bl, lambda_boundary=0.0) + print("weighted loss:", lw.item()) + print("plain mean loss:", lw0.item(), "raw mean:", pl.mean().item()) + + logits = torch.randn(2, 16) + bce = boundary_bce_loss(logits, bl) + print("boundary bce:", bce.item()) diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..4f51fb6 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1 @@ +"""Model package for BA-FPCC-OBB and original FPCC modules.""" diff --git a/models/boundary_branch.py b/models/boundary_branch.py new file mode 100644 index 0000000..5fb0079 --- /dev/null +++ b/models/boundary_branch.py @@ -0,0 +1,43 @@ +"""Simple boundary prediction branch for point features.""" + +import torch +import torch.nn as nn + + +class BoundaryBranch(nn.Module): + def __init__(self, in_channels, hidden_channels=64, input_format="BNC"): + super().__init__() + if input_format not in ["BNC", "BCN"]: + raise ValueError("input_format must be 'BNC' or 'BCN'") + self.input_format = input_format + if input_format == "BNC": + self.net = nn.Sequential( + nn.Linear(in_channels, hidden_channels), + nn.ReLU(inplace=True), + nn.Linear(hidden_channels, 1), + ) + else: + self.net = nn.Sequential( + nn.Conv1d(in_channels, hidden_channels, 1), + nn.ReLU(inplace=True), + nn.Conv1d(hidden_channels, 1, 1), + ) + + def forward(self, x): + if self.input_format == "BNC": + out = self.net(x).squeeze(-1) + else: + out = self.net(x).squeeze(1) + return out + + +if __name__ == "__main__": + x = torch.randn(2, 4096, 128) + m = BoundaryBranch(128, 64, input_format="BNC") + y = m(x) + print("BNC out:", y.shape) + + x2 = x.permute(0, 2, 1) + m2 = BoundaryBranch(128, 64, input_format="BCN") + y2 = m2(x2) + print("BCN out:", y2.shape) diff --git a/models/fpcc_pytorch.py b/models/fpcc_pytorch.py new file mode 100644 index 0000000..7fdd2aa --- /dev/null +++ b/models/fpcc_pytorch.py @@ -0,0 +1,72 @@ +"""PyTorch FPCC backbone/head (simple readable version) for migration.""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class SimplePointMLP(nn.Module): + def __init__(self, in_dim=6, feat_dim=128): + super().__init__() + self.net = nn.Sequential( + nn.Linear(in_dim, 64), + nn.ReLU(inplace=True), + nn.Linear(64, 128), + nn.ReLU(inplace=True), + nn.Linear(128, feat_dim), + ) + + def forward(self, x): + # x: [B,N,C] + return self.net(x) + + +class FPCCNetTorch(nn.Module): + def __init__(self, in_dim=6, feat_dim=128): + super().__init__() + self.backbone = SimplePointMLP(in_dim=in_dim, feat_dim=feat_dim) + self.center_head = nn.Sequential( + nn.Linear(feat_dim, 64), + nn.ReLU(inplace=True), + nn.Linear(64, 1), + ) + + def forward(self, points): + # points: [B,N,C] + feat = self.backbone(points) + center_logits = self.center_head(feat).squeeze(-1) + center_score = torch.sigmoid(center_logits) + + # pairwise feature distance as similarity matrix logits + dist_feat = torch.cdist(feat, feat, p=2) ** 2 + return { + "point_features": feat, + "center_logits": center_logits, + "center_score": center_score, + "simmat": dist_feat, + } + + +def fpcc_loss_torch(net_output, group_one_hot, center_labels, margin_same=0.5, margin_diff=1.0): + """Approximate FPCC loss in PyTorch. + + group_one_hot: [B,N,G] + center_labels: [B,N] + """ + sim = net_output["simmat"] + B, N, _ = sim.shape + + group_mat = torch.matmul(group_one_hot, group_one_hot.transpose(1, 2)) + eye = torch.eye(N, device=sim.device).unsqueeze(0).expand(B, -1, -1) + group_mat = torch.maximum(group_mat, eye) + + same = group_mat + diff = 1.0 - group_mat + + same_loss = 2.0 * same * torch.relu(sim - margin_same) + diff_loss = diff * torch.relu(margin_diff - sim) + simmat_loss = (same_loss + diff_loss).mean() + + center_loss = F.smooth_l1_loss(net_output["center_score"], center_labels) + total = simmat_loss + 3.0 * center_loss + return total, center_loss, simmat_loss diff --git a/scripts/convert_syn_pbox_one_frame.py b/scripts/convert_syn_pbox_one_frame.py new file mode 100644 index 0000000..9db6a5f --- /dev/null +++ b/scripts/convert_syn_pbox_one_frame.py @@ -0,0 +1,105 @@ +"""Convert one SYN-PBOX frame to BA-FPCC npz with boundary labels.""" + +import argparse +import glob +import json +import os + +import imageio.v2 as imageio +import numpy as np + +from utils.boundary_utils import generate_boundary_label + + +def find_masks(mask_dir, frame_id): + patterns = [ + os.path.join(mask_dir, f"{frame_id:06d}_*.png"), + os.path.join(mask_dir, f"{frame_id:04d}_*.png"), + os.path.join(mask_dir, f"{frame_id}_*.png"), + ] + files = [] + for p in patterns: + files.extend(glob.glob(p)) + files = sorted(list(set(files))) + if not files: + raise FileNotFoundError(f"No mask found for frame_id={frame_id} in {mask_dir}") + return files + + +def backproject_depth(depth, K, depth_scale): + h, w = depth.shape + u, v = np.meshgrid(np.arange(w), np.arange(h)) # u: x(pixel col), v: y(pixel row) + z = depth.astype(np.float32) / float(depth_scale) + fx, fy = K[0, 0], K[1, 1] + cx, cy = K[0, 2], K[1, 2] + x = (u - cx) * z / fx + y = (v - cy) * z / fy + pts = np.stack([x, y, z], axis=-1) + return pts + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--scene_camera", required=True) + ap.add_argument("--depth", required=True) + ap.add_argument("--mask_dir", required=True) + ap.add_argument("--frame_id", type=int, default=0) + ap.add_argument("--output", required=True) + ap.add_argument("--num_points", type=int, default=4096) + args = ap.parse_args() + + with open(args.scene_camera, "r", encoding="utf-8") as f: + scene_camera = json.load(f) + + frame_key = str(args.frame_id) + if frame_key not in scene_camera: + raise KeyError(f"frame_id={args.frame_id} not found in scene_camera") + + cam = scene_camera[frame_key] + K = np.array(cam["cam_K"], dtype=np.float32).reshape(3, 3) + depth_scale = float(cam.get("depth_scale", 1000.0)) + + depth = imageio.imread(args.depth) + pts_img = backproject_depth(depth, K, depth_scale) + + masks = find_masks(args.mask_dir, args.frame_id) + instance_map = np.full(depth.shape, -1, dtype=np.int32) + for ins_id, mpath in enumerate(masks): + m = imageio.imread(mpath) + instance_map[m > 0] = ins_id + + valid = depth > 0 + points = pts_img[valid] + instance_labels = instance_map[valid] + + if points.shape[0] == 0: + raise RuntimeError("No valid depth points after filtering depth>0") + + n = points.shape[0] + if n >= args.num_points: + idx = np.random.choice(n, args.num_points, replace=False) + else: + idx = np.random.choice(n, args.num_points, replace=True) + + points = points[idx] + instance_labels = instance_labels[idx] + + center_labels = np.zeros((args.num_points,), dtype=np.float32) + for ins in np.unique(instance_labels): + if ins < 0: + continue + ins_idx = np.where(instance_labels == ins)[0] + c = points[ins_idx].mean(axis=0) + near = ins_idx[np.argmin(np.linalg.norm(points[ins_idx] - c, axis=1))] + center_labels[near] = 1.0 + + boundary_labels = generate_boundary_label(points, instance_labels, k=16, ignore_label=-1).astype(np.float32) + + np.savez(args.output, points=points, instance_labels=instance_labels, center_labels=center_labels, boundary_labels=boundary_labels) + print("Saved:", args.output) + for ins in np.unique(instance_labels): + print(f"instance {ins}: {(instance_labels==ins).sum()} points") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_ba_fpcc.py b/scripts/test_ba_fpcc.py new file mode 100644 index 0000000..f5b8473 --- /dev/null +++ b/scripts/test_ba_fpcc.py @@ -0,0 +1,144 @@ +"""Test BA-FPCC-OBB in PyTorch and run FPCC-style clustering + optional OBB.""" + +import argparse +import glob +import json +import os +import numpy as np +import torch + +from models.fpcc_pytorch import FPCCNetTorch +from models.boundary_branch import BoundaryBranch +from utils.test_utils import GroupMerging_fpcc +from utils.obb_utils import estimate_obbs_for_instances +from utils.simple_visualize import visualize_instances_with_obbs + + +def samples(data, sample_num_point): + n, dim = data.shape + # keep original order to avoid prediction-index mismatch + bnum = int(np.ceil(n / float(sample_num_point))) + out = np.zeros((bnum, sample_num_point, dim), dtype=np.float32) + for i in range(bnum): + beg = i * sample_num_point + end = min((i + 1) * sample_num_point, n) + num = end - beg + out[i, :num] = data[beg:end] + if num < sample_num_point: + idx = np.random.choice(n, sample_num_point - num) + out[i, num:] = data[idx] + return out + + +def reassign_boundary_points(points, pred_instance_labels, boundary_scores, threshold=0.5): + labels = pred_instance_labels.copy() + bmask = boundary_scores > threshold + valid = (~bmask) & (labels >= 0) + if valid.sum() == 0: + return labels + centers = {} + for i in np.unique(labels[valid]): + centers[int(i)] = points[(labels == i) & valid].mean(axis=0) + if len(centers) == 0: + return labels + cids = np.array(list(centers.keys())) + cpts = np.array([centers[i] for i in cids]) + idx = np.where(bmask)[0] + if idx.size == 0: + return labels + d2 = ((points[idx, None, :] - cpts[None, :, :]) ** 2).sum(axis=2) + nn = d2.argmin(axis=1) + labels[idx] = cids[nn] + return labels + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ckpt", type=str, default="checkpoint/ba_fpcc_torch.pt") + ap.add_argument("--test_glob", type=str, default="datas/ring_test/*.txt") + ap.add_argument("--point_dim", type=int, default=6) + ap.add_argument("--point_num", type=int, default=4096) + ap.add_argument("--center_score_th", type=float, default=0.6) + ap.add_argument("--r_nms", type=float, default=0.1) + ap.add_argument("--use_boundary_branch", action="store_true") + ap.add_argument("--boundary_th", type=float, default=0.5) + ap.add_argument("--use_obb", action="store_true") + ap.add_argument("--save_obb", action="store_true") + ap.add_argument("--visualize_obb", action="store_true") + ap.add_argument("--obb_json", type=str, default="test_results/ba_fpcc_obb.json") + args = ap.parse_args() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = FPCCNetTorch(in_dim=args.point_dim, feat_dim=128).to(device) + bbranch = BoundaryBranch(128, 64, input_format="BNC").to(device) if args.use_boundary_branch else None + + state = torch.load(args.ckpt, map_location=device) + model.load_state_dict(state["model"]) + if bbranch is not None and state.get("boundary_branch") is not None: + bbranch.load_state_dict(state["boundary_branch"]) + + model.eval() + if bbranch is not None: + bbranch.eval() + + os.makedirs("test_results", exist_ok=True) + obb_rows = [] + + test_files = sorted(glob.glob(args.test_glob)) + if len(test_files) == 0: + raise FileNotFoundError(f"No test files matched: {args.test_glob}") + + for fp in test_files: + scene_id = os.path.basename(fp).split(".")[0] + raw = np.loadtxt(fp) + pts_xyz = raw[:, :3] + ins_gt = raw[:, -1].astype(np.int32) + + pts = np.zeros((pts_xyz.shape[0], 6), dtype=np.float32) + pts[:, :3] = pts_xyz + pts[:, 3:6] = pts_xyz - pts_xyz.min(axis=0, keepdims=True) + + batches = samples(pts, args.point_num) + with torch.no_grad(): + x = torch.from_numpy(batches[:, :, : args.point_dim]).float().to(device) + out = model(x) + feat = out["point_features"].cpu().numpy().reshape(-1, 128)[: pts.shape[0]] + cscore = out["center_score"].cpu().numpy().reshape(-1, 1)[: pts.shape[0]] + bscore = None + if bbranch is not None: + blogits = bbranch(out["point_features"]) + bscore = torch.sigmoid(blogits).cpu().numpy().reshape(-1)[: pts.shape[0]] + + pred_ins, _ = GroupMerging_fpcc(pts[:, 3:6], feat, cscore, center_socre_th=args.center_score_th, r_nms=args.r_nms) + pred_ins = pred_ins.astype(np.int32) + if bscore is not None: + pred_ins = reassign_boundary_points(pts[:, :3], pred_ins, bscore, threshold=args.boundary_th) + + if args.use_obb: + obb_list = estimate_obbs_for_instances(pts[:, :3], pred_ins) + if args.save_obb: + for o in obb_list: + obb_rows.append({ + "scene_id": scene_id, + "instance_id": int(o["instance_id"]), + "center": np.asarray(o["center"]).tolist(), + "extent": np.asarray(o["extent"]).tolist(), + "R": np.asarray(o["R"]).tolist(), + "yaw": float(o["yaw"]), + "num_points": int(o["num_points"]), + }) + if args.visualize_obb: + visualize_instances_with_obbs(pts[:, :3], pred_ins, obb_list) + + out_txt = os.path.join("test_results", f"{scene_id}_pred.txt") + np.savetxt(out_txt, np.column_stack([pts[:, :3], pred_ins]), fmt="%.6f %.6f %.6f %d") + print("saved pred:", out_txt, "gt instances:", len(np.unique(ins_gt)), "pred instances:", len(np.unique(pred_ins))) + + if args.use_obb and args.save_obb: + with open(args.obb_json, "w", encoding="utf-8") as f: + json.dump(obb_rows, f, indent=2) + print("saved obb:", args.obb_json) + + +if __name__ == "__main__": + main() diff --git a/scripts/train_ba_fpcc.py b/scripts/train_ba_fpcc.py new file mode 100644 index 0000000..f314c84 --- /dev/null +++ b/scripts/train_ba_fpcc.py @@ -0,0 +1,91 @@ +"""Train BA-FPCC-OBB in PyTorch using original FPCC dataset format.""" + +import argparse +import numpy as np +import torch +from torch.utils.data import DataLoader + +from datasets.fpcc_dataset import FPCCDataset +from models.fpcc_pytorch import FPCCNetTorch, fpcc_loss_torch +from models.boundary_branch import BoundaryBranch +from losses.boundary_loss import boundary_bce_loss +from utils.boundary_utils import generate_boundary_label + + +def build_boundary_labels(points_bnc, instance_labels_bn): + b, n, _ = points_bnc.shape + out = np.zeros((b, n), dtype=np.float32) + for i in range(b): + out[i] = generate_boundary_label(points_bnc[i], instance_labels_bn[i], k=16, ignore_label=-1) + return torch.from_numpy(out) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--input_list", type=str, default="datas/ring_train.txt") + ap.add_argument("--batch_size", type=int, default=4) + ap.add_argument("--epochs", type=int, default=10) + ap.add_argument("--point_dim", type=int, default=6) + ap.add_argument("--num_groups", type=int, default=50) + ap.add_argument("--lr", type=float, default=1e-4) + ap.add_argument("--save_path", type=str, default="checkpoint/ba_fpcc_torch.pt") + ap.add_argument("--use_boundary_loss", action="store_true") + ap.add_argument("--use_boundary_branch", action="store_true") + ap.add_argument("--lambda_boundary", type=float, default=1.0) + ap.add_argument("--lambda_bce", type=float, default=0.5) + args = ap.parse_args() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if not torch.cuda.is_available(): + print("[WARN] CUDA not available, using CPU") + ds = FPCCDataset(args.input_list, point_dim=args.point_dim, num_groups=args.num_groups) + dl = DataLoader(ds, batch_size=args.batch_size, shuffle=True, num_workers=0) + + model = FPCCNetTorch(in_dim=args.point_dim, feat_dim=128).to(device) + bbranch = BoundaryBranch(128, 64, input_format="BNC").to(device) if args.use_boundary_branch else None + + params = list(model.parameters()) + (list(bbranch.parameters()) if bbranch is not None else []) + opt = torch.optim.Adam(params, lr=args.lr) + + for epoch in range(args.epochs): + model.train() + if bbranch is not None: + bbranch.train() + running = 0.0 + for batch in dl: + points = batch["points"].float().to(device) + group_one_hot = batch["group_one_hot"].float().to(device) + center_labels = batch["center_labels"].float().to(device) + instance_labels = batch["instance_labels"].cpu().numpy().astype(np.int32) + + out = model(points) + total_loss, center_loss, sim_loss = fpcc_loss_torch(out, group_one_hot, center_labels) + + boundary_labels = None + if args.use_boundary_loss or args.use_boundary_branch: + boundary_labels = build_boundary_labels(points.detach().cpu().numpy()[:, :, :3], instance_labels).to(device) + + if args.use_boundary_loss: + weights = 1.0 + args.lambda_boundary * boundary_labels + weighted_center = ((out["center_score"] - center_labels).abs() * weights).mean() + total_loss = total_loss + weighted_center + + if args.use_boundary_branch: + logits = bbranch(out["point_features"]) + bce = boundary_bce_loss(logits, boundary_labels) + total_loss = total_loss + args.lambda_bce * bce + + opt.zero_grad() + total_loss.backward() + opt.step() + + running += float(total_loss.item()) + + print(f"Epoch {epoch+1}/{args.epochs} loss={running/max(len(dl),1):.6f}") + + torch.save({"model": model.state_dict(), "boundary_branch": None if bbranch is None else bbranch.state_dict()}, args.save_path) + print("Saved checkpoint:", args.save_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/visualize_syn_pbox_one_frame.py b/scripts/visualize_syn_pbox_one_frame.py new file mode 100644 index 0000000..7e53fbe --- /dev/null +++ b/scripts/visualize_syn_pbox_one_frame.py @@ -0,0 +1,27 @@ +"""Visualize one converted SYN-PBOX frame with instances/boundary/centers/optional OBB.""" + +import argparse +import numpy as np + +from utils.obb_utils import estimate_obbs_for_instances +from utils.simple_visualize import visualize_instances_with_obbs + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--npz", required=True) + ap.add_argument("--show_obb", action="store_true") + args = ap.parse_args() + + data = np.load(args.npz) + points = data["points"] + instance_labels = data["instance_labels"] + boundary_labels = data["boundary_labels"] if "boundary_labels" in data else None + center_labels = data["center_labels"] if "center_labels" in data else None + + obb_list = estimate_obbs_for_instances(points, instance_labels) if args.show_obb else [] + visualize_instances_with_obbs(points, instance_labels, obb_list, boundary_labels, center_labels) + + +if __name__ == "__main__": + main() diff --git a/utils/boundary_utils.py b/utils/boundary_utils.py new file mode 100644 index 0000000..c7ee65e --- /dev/null +++ b/utils/boundary_utils.py @@ -0,0 +1,57 @@ +"""Boundary label generation utilities for BA-FPCC-OBB.""" + +import numpy as np +from sklearn.neighbors import NearestNeighbors + + +def generate_boundary_label(points, instance_labels, k=16, ignore_label=-1): + """Generate binary boundary labels from instance ids. + + Args: + points (np.ndarray): [N, 3] + instance_labels (np.ndarray): [N] + k (int): number of nearest neighbors + ignore_label (int): ignored instance id, boundary set to 0 + + Returns: + np.ndarray: boundary labels [N], values in {0, 1} + """ + points = np.asarray(points) + instance_labels = np.asarray(instance_labels) + if points.ndim != 2 or points.shape[1] != 3: + raise ValueError(f"points shape must be [N,3], got {points.shape}") + if instance_labels.ndim != 1 or instance_labels.shape[0] != points.shape[0]: + raise ValueError("instance_labels shape must be [N] and match points") + + n = points.shape[0] + if n == 0: + return np.zeros((0,), dtype=np.uint8) + + k_eff = min(max(2, k), n) + nn = NearestNeighbors(n_neighbors=k_eff, algorithm="auto") + nn.fit(points) + indices = nn.kneighbors(points, return_distance=False) + + boundary = np.zeros((n,), dtype=np.uint8) + for i in range(n): + cur_label = instance_labels[i] + if cur_label == ignore_label: + continue + nbr_labels = instance_labels[indices[i]] + nbr_labels = nbr_labels[nbr_labels != ignore_label] + if nbr_labels.size == 0: + continue + if np.any(nbr_labels != cur_label): + boundary[i] = 1 + return boundary + + +if __name__ == "__main__": + np.random.seed(0) + a = np.random.randn(200, 3) * 0.02 + np.array([0.0, 0.0, 0.0]) + b = np.random.randn(200, 3) * 0.02 + np.array([0.05, 0.0, 0.0]) + pts = np.vstack([a, b]) + ins = np.array([0] * 200 + [1] * 200) + bd = generate_boundary_label(pts, ins, k=16) + print("Total points:", len(pts)) + print("Boundary points:", int(bd.sum())) diff --git a/utils/obb_utils.py b/utils/obb_utils.py new file mode 100644 index 0000000..c19193b --- /dev/null +++ b/utils/obb_utils.py @@ -0,0 +1,48 @@ +"""Open3D based OBB estimation utilities for BA-FPCC-OBB.""" + +import numpy as np +import open3d as o3d + + +def estimate_obb(points): + """Estimate oriented bounding box for points [N,3].""" + points = np.asarray(points) + if points.ndim != 2 or points.shape[1] != 3: + raise ValueError(f"points must be [N,3], got {points.shape}") + if points.shape[0] < 4: + return None + + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(points) + obb = pcd.get_oriented_bounding_box() + center = np.asarray(obb.center) + extent = np.asarray(obb.extent) + R = np.asarray(obb.R) + # TODO: yaw 依赖坐标系定义,这里按 Z-up 假设从 R 提取平面角。 + yaw = float(np.arctan2(R[1, 0], R[0, 0])) + return {"center": center, "extent": extent, "R": R, "yaw": yaw} + + +def estimate_obbs_for_instances(points, instance_labels): + points = np.asarray(points) + instance_labels = np.asarray(instance_labels) + out = [] + for ins_id in np.unique(instance_labels): + if ins_id < 0: + continue + mask = instance_labels == ins_id + ins_points = points[mask] + obb = estimate_obb(ins_points) + if obb is None: + continue + obb["instance_id"] = int(ins_id) + obb["num_points"] = int(ins_points.shape[0]) + out.append(obb) + return out + + +if __name__ == "__main__": + np.random.seed(0) + pts = np.random.rand(1000, 3) * np.array([0.4, 0.2, 0.1]) + obb = estimate_obb(pts) + print("obb:", obb) diff --git a/utils/simple_metrics.py b/utils/simple_metrics.py new file mode 100644 index 0000000..d3f99fb --- /dev/null +++ b/utils/simple_metrics.py @@ -0,0 +1,64 @@ +"""Simple metrics for BA-FPCC-OBB experiments.""" + +import time +import numpy as np +import torch + + +def compute_iou(pred_mask, gt_mask): + pred = np.asarray(pred_mask).astype(bool) + gt = np.asarray(gt_mask).astype(bool) + inter = np.logical_and(pred, gt).sum() + union = np.logical_or(pred, gt).sum() + return float(inter / union) if union > 0 else 0.0 + + +def compute_precision_recall_f1(pred_labels, gt_labels): + pred = np.asarray(pred_labels).reshape(-1) + gt = np.asarray(gt_labels).reshape(-1) + tp = np.sum((pred == 1) & (gt == 1)) + fp = np.sum((pred == 1) & (gt == 0)) + fn = np.sum((pred == 0) & (gt == 1)) + precision = tp / (tp + fp + 1e-8) + recall = tp / (tp + fn + 1e-8) + f1 = 2 * precision * recall / (precision + recall + 1e-8) + return float(precision), float(recall), float(f1) + + +def compute_boundary_f1(pred_boundary, gt_boundary): + _, _, f1 = compute_precision_recall_f1(pred_boundary, gt_boundary) + return f1 + + +def compute_center_error(pred_center, gt_center): + pred = np.asarray(pred_center) + gt = np.asarray(gt_center) + return float(np.linalg.norm(pred - gt)) + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +def measure_fps(model, dataloader, num_batches=20): + model.eval() + use_cuda = next(model.parameters()).is_cuda if any(True for _ in model.parameters()) else False + t0 = time.time() + cnt = 0 + with torch.no_grad(): + for i, batch in enumerate(dataloader): + if i >= num_batches: + break + x = batch[0] if isinstance(batch, (list, tuple)) else batch + if use_cuda: + x = x.cuda(non_blocking=True) + _ = model(x) + cnt += x.shape[0] + dt = max(time.time() - t0, 1e-8) + return float(cnt / dt) + + +if __name__ == "__main__": + print("iou:", compute_iou([1, 1, 0], [1, 0, 0])) + print("f1:", compute_precision_recall_f1([1, 0, 1], [1, 1, 0])) + # TODO: mAP for instance segmentation if needed in future. diff --git a/utils/simple_visualize.py b/utils/simple_visualize.py new file mode 100644 index 0000000..a1b97e5 --- /dev/null +++ b/utils/simple_visualize.py @@ -0,0 +1,52 @@ +"""Simple Open3D visualization helpers for instances, boundaries, centers and OBBs.""" + +import numpy as np +import open3d as o3d + + +def _color_by_instance(instance_labels): + uniq = np.unique(instance_labels) + color_map = {} + rng = np.random.RandomState(0) + for u in uniq: + if u < 0: + color_map[u] = np.array([0.5, 0.5, 0.5]) + else: + color_map[u] = rng.rand(3) + colors = np.array([color_map[i] for i in instance_labels], dtype=np.float64) + return colors + + +def visualize_instances_with_obbs(points, instance_labels, obb_list, boundary_labels=None, center_labels=None): + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(points) + pcd.colors = o3d.utility.Vector3dVector(_color_by_instance(instance_labels)) + + geoms = [pcd] + + if boundary_labels is not None: + bmask = boundary_labels.astype(bool) + if bmask.any(): + bpcd = o3d.geometry.PointCloud() + bpcd.points = o3d.utility.Vector3dVector(points[bmask]) + bpcd.paint_uniform_color([1.0, 0.0, 0.0]) + geoms.append(bpcd) + + if center_labels is not None: + cmask = center_labels > 0.5 + if cmask.any(): + cpcd = o3d.geometry.PointCloud() + cpcd.points = o3d.utility.Vector3dVector(points[cmask]) + cpcd.paint_uniform_color([0.0, 1.0, 0.0]) + geoms.append(cpcd) + + for obb_item in obb_list: + obb = o3d.geometry.OrientedBoundingBox( + center=np.array(obb_item["center"]), + R=np.array(obb_item["R"]), + extent=np.array(obb_item["extent"]), + ) + obb.color = (0.0, 0.0, 0.0) + geoms.append(obb) + + o3d.visualization.draw_geometries(geoms) From 1bba76d7050f096c831bbbfc0a436556145a2aff Mon Sep 17 00:00:00 2001 From: RY150150 <87613708+RY150150@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:40:11 +0800 Subject: [PATCH 2/2] Expand README with full setup, runbook, outputs and troubleshooting --- README_BA_FPCC.md | 188 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 171 insertions(+), 17 deletions(-) diff --git a/README_BA_FPCC.md b/README_BA_FPCC.md index 46afea9..5ed22e0 100644 --- a/README_BA_FPCC.md +++ b/README_BA_FPCC.md @@ -1,25 +1,103 @@ # BA-FPCC-OBB -本版本提供 **PyTorch 主干迁移版**(不改坏原 FPCC TensorFlow 代码)。 +本 README 是对当前仓库里 BA-FPCC-OBB 改造的**完整运行说明**,目标是: +- 不破坏原始 FPCC(TensorFlow)流程; +- 提供可独立运行的 PyTorch 迁移流程; +- 支持 Boundary Label / Boundary Loss / Boundary Branch / Boundary-aware 后处理 / OBB / SYN-PBOX 单帧转换与可视化。 + +--- + +## 0. 分支与安全策略(强烈推荐) + +为了确保原始 FPCC 不受影响,请在独立分支开发与验证: -## 安全建议(强烈推荐) -先在新分支开发,避免影响原始 FPCC: ```bash git checkout -b feature/ba-fpcc-pytorch-migration ``` -## 1) 原始 FPCC baseline(保留不变) +- 原始 TF 入口:`fpcc_train.py`、`fpcc_test.py` +- 新增 PyTorch 入口:`scripts/train_ba_fpcc.py`、`scripts/test_ba_fpcc.py` + +--- + +## 1. 目录说明(新增模块) + +- `datasets/fpcc_dataset.py`:读取原 FPCC h5 列表文件,输出 PyTorch batch 字段。 +- `models/fpcc_pytorch.py`:PyTorch FPCC 主干 + FPCC 风格损失。 +- `models/boundary_branch.py`:边界预测分支。 +- `losses/boundary_loss.py`:boundary 加权损失与 BCE 损失。 +- `utils/boundary_utils.py`:基于 kNN 的 boundary label 自动生成。 +- `utils/obb_utils.py`:Open3D OBB 估计。 +- `utils/simple_visualize.py`:实例/边界/中心/OBB 可视化。 +- `utils/simple_metrics.py`:简化指标函数。 +- `scripts/train_ba_fpcc.py`:PyTorch 训练入口(含 BA 开关)。 +- `scripts/test_ba_fpcc.py`:PyTorch 测试入口(含后处理与 OBB 开关)。 +- `scripts/convert_syn_pbox_one_frame.py`:SYN-PBOX 单帧转换。 +- `scripts/visualize_syn_pbox_one_frame.py`:SYN-PBOX 单帧可视化。 + +--- + +## 2. 环境依赖 + +> 你当前机器如果提示 `ModuleNotFoundError`,先执行这一节。 + +### 2.1 PyTorch 分支依赖 + +```bash +pip install numpy scipy h5py scikit-learn imageio open3d +pip install torch torchvision torchaudio +``` + +### 2.2 原始 FPCC(TensorFlow)依赖 + +原仓库是 TensorFlow 1.x 风格代码,建议单独环境安装: + +```bash +pip install numpy scipy h5py +pip install tensorflow==1.15.5 +``` + +> 若你的系统无法安装 TF1,请只跑 PyTorch 分支,原始分支作为结构参考。 + +--- + +## 3. 数据格式说明 + +### 3.1 PyTorch 训练数据(与原 FPCC 一致) + +`--input_list` 指向一个 txt 文件(如 `datas/ring_train.txt`),每行是一个 h5 文件路径。h5 至少包含: +- `data`:点云输入 +- `pid` 或 `gid`:实例 id +- `score` 或 `center_score`:中心监督标签 + +### 3.2 PyTorch 测试数据 + +默认读取 `datas/ring_test/*.txt`,每行格式近似: + +```text +x y z instance_id +``` + +--- + +## 4. 运行命令 + +## 4.1 原始 FPCC baseline(TF) + ```bash python fpcc_train.py --gpu 0 --input_list datas/ring_train.txt python fpcc_test.py --gpu 0 --restore_dir checkpoint/ ``` -## 2) PyTorch 版 BA-FPCC 训练 +## 4.2 PyTorch BA-FPCC 训练 + ```bash python scripts/train_ba_fpcc.py \ --input_list datas/ring_train.txt \ --batch_size 4 \ --epochs 100 \ + --point_dim 6 \ + --num_groups 50 \ --use_boundary_loss \ --use_boundary_branch \ --lambda_boundary 1.0 \ @@ -27,37 +105,113 @@ python scripts/train_ba_fpcc.py \ --save_path checkpoint/ba_fpcc_torch.pt ``` -## 3) PyTorch 版 BA-FPCC 测试 + OBB +## 4.3 PyTorch BA-FPCC 测试 + Boundary-aware + OBB + ```bash python scripts/test_ba_fpcc.py \ --ckpt checkpoint/ba_fpcc_torch.pt \ --test_glob 'datas/ring_test/*.txt' \ + --point_dim 6 \ + --point_num 4096 \ + --center_score_th 0.6 \ + --r_nms 0.1 \ --use_boundary_branch \ - --use_obb --save_obb \ + --boundary_th 0.5 \ + --use_obb \ + --save_obb \ --obb_json test_results/ba_fpcc_obb.json ``` -## 4) SYN-PBOX 单帧转换 +## 4.4 SYN-PBOX 单帧转换 + ```bash python scripts/convert_syn_pbox_one_frame.py \ --scene_camera path/to/scene_camera.json \ --depth path/to/depth.png \ --mask_dir path/to/mask_visib \ --frame_id 0 \ - --output demo_syn_frame.npz + --output demo_syn_frame.npz \ + --num_points 4096 ``` -## 5) SYN-PBOX 单帧可视化 +## 4.5 SYN-PBOX 单帧可视化 + ```bash python scripts/visualize_syn_pbox_one_frame.py --npz demo_syn_frame.npz --show_obb ``` -## 说明 -- 原始 TF 训练/测试入口未删除,便于回归对照。 -- 新增 PyTorch 训练/测试脚本已使用原 FPCC 数据格式(h5/txt)并包含 BA 开关。 +--- +## 5. 开关与消融建议 -## 环境依赖 -```bash -pip install numpy torch scikit-learn open3d imageio h5py scipy -``` +### 5.1 训练开关(`scripts/train_ba_fpcc.py`) +- `--use_boundary_loss`:开启 boundary 加权项。 +- `--use_boundary_branch`:开启 boundary branch + BCE。 +- `--lambda_boundary`:boundary 加权强度。 +- `--lambda_bce`:boundary BCE 损失权重。 + +建议消融顺序: +1. 仅主干(两个开关都关) +2. 主干 + boundary loss +3. 主干 + boundary branch +4. 全开(boundary loss + boundary branch) + +### 5.2 测试开关(`scripts/test_ba_fpcc.py`) +- `--use_boundary_branch`:加载并使用边界分数。 +- `--boundary_th`:边界点阈值。 +- `--use_obb`:计算 OBB。 +- `--save_obb`:保存 OBB json。 +- `--visualize_obb`:Open3D 交互可视化。 + +--- + +## 6. 输出说明 + +### 6.1 测试输出 +- `test_results/*_pred.txt`:每场景预测实例结果。 +- `test_results/ba_fpcc_obb.json`:OBB 列表(当 `--use_obb --save_obb` 开启时)。 + +OBB 字段包含: +- `scene_id` +- `instance_id` +- `center` +- `extent` +- `R` +- `yaw` +- `num_points` + +### 6.2 SYN-PBOX NPZ 输出 +`convert_syn_pbox_one_frame.py` 输出字段: +- `points` +- `instance_labels` +- `center_labels` +- `boundary_labels` + +--- + +## 7. 常见错误与排查 + +### 7.1 `ModuleNotFoundError: numpy / torch / tensorflow` +先安装第 2 节依赖。 + +### 7.2 `No test files matched` +检查 `--test_glob` 路径与引号是否正确。 + +### 7.3 OBB 可视化打不开 +检查 `open3d` 是否安装;无桌面环境时不要开 `--visualize_obb`,只保存 json。 + +### 7.4 TF1 安装困难 +建议优先使用 PyTorch 分支;原 TF 分支可作为对照基线。 + +--- + +## 8. 当前实现边界(实话说明) + +- PyTorch 主干是“可读优先”的迁移版本,不是逐层 1:1 复刻 TF 主干。 +- 若你要严格复现实验数值,需要继续做: + 1) backbone 细节对齐; + 2) loss 细节对齐; + 3) 数据预处理完全对齐。 + +但从工程可运行性上,当前分支已经具备完整链路: +**数据读取 -> 训练 -> 测试聚类 -> boundary 后处理 -> OBB 输出 -> 可视化**。