diff --git a/mart/configs/batch_c15n/input_tensor_float01.yaml b/mart/configs/batch_c15n/input_tensor_float01.yaml new file mode 100644 index 00000000..972a1e4f --- /dev/null +++ b/mart/configs/batch_c15n/input_tensor_float01.yaml @@ -0,0 +1,5 @@ +# Scale pixel input in the range of [0,1] to [0,255] for Adversary. +defaults: + - tuple + - transform: times_255_and_round + - transform@untransform: divided_by_255 diff --git a/mart/configs/batch_c15n/input_tuple_float01.yaml b/mart/configs/batch_c15n/input_tuple_float01.yaml new file mode 100644 index 00000000..49dbe4c2 --- /dev/null +++ b/mart/configs/batch_c15n/input_tuple_float01.yaml @@ -0,0 +1,11 @@ +# Scale pixel input in the range of [0,1] to [0,255] for Adversary. +defaults: + - tuple + - transform@transform.transforms: times_255_and_round + - transform@untransform.transforms: divided_by_255 + +transform: + _target_: mart.transforms.TupleTransforms + +untransform: + _target_: mart.transforms.TupleTransforms diff --git a/mart/configs/batch_c15n/transform/divided_by_255.yaml b/mart/configs/batch_c15n/transform/divided_by_255.yaml new file mode 100644 index 00000000..92a63b7c --- /dev/null +++ b/mart/configs/batch_c15n/transform/divided_by_255.yaml @@ -0,0 +1,3 @@ +_target_: torchvision.transforms.Normalize +mean: 0 +std: 255 diff --git a/mart/configs/batch_c15n/transform/times_255_and_round.yaml b/mart/configs/batch_c15n/transform/times_255_and_round.yaml new file mode 100644 index 00000000..dbeff64d --- /dev/null +++ b/mart/configs/batch_c15n/transform/times_255_and_round.yaml @@ -0,0 +1,13 @@ +_target_: torchvision.transforms.Compose +transforms: + - _target_: mart.transforms.Denormalize + center: 0 + scale: 255 + # Fix potential numeric error. + - _target_: torch.fake_quantize_per_tensor_affine + _partial_: true + # (x/1+0).round().clamp(0, 255) * 1 + scale: 1 + zero_point: 0 + quant_min: 0 + quant_max: 255 diff --git a/mart/configs/datamodule/cifar10.yaml b/mart/configs/datamodule/cifar10.yaml index 9392448b..6988096b 100644 --- a/mart/configs/datamodule/cifar10.yaml +++ b/mart/configs/datamodule/cifar10.yaml @@ -17,16 +17,6 @@ train_dataset: padding_mode: reflect - _target_: torchvision.transforms.RandomHorizontalFlip - _target_: torchvision.transforms.ToTensor - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 target_transform: null download: true @@ -38,16 +28,6 @@ val_dataset: _target_: torchvision.transforms.Compose transforms: - _target_: torchvision.transforms.ToTensor - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 target_transform: null download: true diff --git a/mart/configs/datamodule/coco.yaml b/mart/configs/datamodule/coco.yaml index a4ec3403..0ea6bccd 100644 --- a/mart/configs/datamodule/coco.yaml +++ b/mart/configs/datamodule/coco.yaml @@ -12,16 +12,6 @@ train_dataset: - _target_: mart.transforms.ConvertCocoPolysToMask - _target_: mart.transforms.RandomHorizontalFlip p: 0.5 - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 val_dataset: _target_: mart.datamodules.coco.CocoDetection @@ -32,16 +22,6 @@ val_dataset: transforms: - _target_: torchvision.transforms.ToTensor - _target_: mart.transforms.ConvertCocoPolysToMask - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 test_dataset: _target_: mart.datamodules.coco.CocoDetection @@ -52,16 +32,6 @@ test_dataset: transforms: - _target_: torchvision.transforms.ToTensor - _target_: mart.transforms.ConvertCocoPolysToMask - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 num_workers: 2 collate_fn: diff --git a/mart/configs/datamodule/coco_perturbable_mask.yaml b/mart/configs/datamodule/coco_perturbable_mask.yaml index 6d7bdd77..461ac9b1 100644 --- a/mart/configs/datamodule/coco_perturbable_mask.yaml +++ b/mart/configs/datamodule/coco_perturbable_mask.yaml @@ -10,16 +10,6 @@ train_dataset: - _target_: mart.transforms.RandomHorizontalFlip p: 0.5 - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 val_dataset: transforms: @@ -28,16 +18,6 @@ val_dataset: # ConvertCocoPolysToMask must be prior to ConvertInstanceSegmentationToPerturbable. - _target_: mart.transforms.ConvertCocoPolysToMask - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 test_dataset: transforms: @@ -46,13 +26,3 @@ test_dataset: # ConvertCocoPolysToMask must be prior to ConvertInstanceSegmentationToPerturbable. - _target_: mart.transforms.ConvertCocoPolysToMask - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 diff --git a/mart/configs/datamodule/fiftyone.yaml b/mart/configs/datamodule/fiftyone.yaml index 1fbdd9a6..7c45687c 100644 --- a/mart/configs/datamodule/fiftyone.yaml +++ b/mart/configs/datamodule/fiftyone.yaml @@ -14,16 +14,6 @@ train_dataset: - _target_: mart.transforms.ConvertCocoPolysToMask - _target_: mart.transforms.RandomHorizontalFlip p: 0.5 - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 val_dataset: _target_: mart.datamodules.fiftyone.FiftyOneDataset @@ -36,16 +26,6 @@ val_dataset: transforms: - _target_: torchvision.transforms.ToTensor - _target_: mart.transforms.ConvertCocoPolysToMask - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 test_dataset: _target_: mart.datamodules.fiftyone.FiftyOneDataset @@ -58,16 +38,6 @@ test_dataset: transforms: - _target_: torchvision.transforms.ToTensor - _target_: mart.transforms.ConvertCocoPolysToMask - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 num_workers: 2 collate_fn: diff --git a/mart/configs/datamodule/fiftyone_perturbable_mask.yaml b/mart/configs/datamodule/fiftyone_perturbable_mask.yaml index 69a7f622..1c6889bd 100644 --- a/mart/configs/datamodule/fiftyone_perturbable_mask.yaml +++ b/mart/configs/datamodule/fiftyone_perturbable_mask.yaml @@ -16,16 +16,6 @@ train_dataset: - _target_: mart.transforms.RandomHorizontalFlip p: 0.5 - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 val_dataset: _target_: mart.datamodules.fiftyone.FiftyOneDataset @@ -40,16 +30,6 @@ val_dataset: # ConvertCocoPolysToMask must be prior to ConvertInstanceSegmentationToPerturbable. - _target_: mart.transforms.ConvertCocoPolysToMask - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 test_dataset: _target_: mart.datamodules.fiftyone.FiftyOneDataset @@ -64,13 +44,3 @@ test_dataset: # ConvertCocoPolysToMask must be prior to ConvertInstanceSegmentationToPerturbable. - _target_: mart.transforms.ConvertCocoPolysToMask - _target_: mart.transforms.ConvertInstanceSegmentationToPerturbable - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 diff --git a/mart/configs/datamodule/imagenet.yaml b/mart/configs/datamodule/imagenet.yaml index 97417bbb..9894a8d7 100644 --- a/mart/configs/datamodule/imagenet.yaml +++ b/mart/configs/datamodule/imagenet.yaml @@ -13,16 +13,6 @@ train_dataset: size: 224 - _target_: torchvision.transforms.RandomHorizontalFlip - _target_: torchvision.transforms.ToTensor - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 val_dataset: _target_: torchvision.datasets.ImageNet @@ -34,16 +24,6 @@ val_dataset: - _target_: torchvision.transforms.CenterCrop size: 224 - _target_: torchvision.transforms.ToTensor - - _target_: mart.transforms.Denormalize - center: 0 - scale: 255 - - _target_: torch.fake_quantize_per_tensor_affine - _partial_: true - # (x/1+0).round().clamp(0, 255) * 1 - scale: 1 - zero_point: 0 - quant_min: 0 - quant_max: 255 test_dataset: ${.val_dataset} diff --git a/mart/configs/experiment/CIFAR10_CNN_Adv.yaml b/mart/configs/experiment/CIFAR10_CNN_Adv.yaml index 72e119da..a867507e 100644 --- a/mart/configs/experiment/CIFAR10_CNN_Adv.yaml +++ b/mart/configs/experiment/CIFAR10_CNN_Adv.yaml @@ -3,6 +3,7 @@ defaults: - /attack@callbacks.adversary_connector.train_adversary: classification_fgsm_linf - /attack@callbacks.adversary_connector.test_adversary: classification_pgd_linf + - override /batch_c15n@callbacks.adversary_connector.batch_c15n: input_tensor_float01 - override /datamodule: cifar10 - override /model: classifier_cifar10_cnn - override /metric: accuracy diff --git a/mart/configs/experiment/COCO_TorchvisionFasterRCNN_Adv.yaml b/mart/configs/experiment/COCO_TorchvisionFasterRCNN_Adv.yaml index 7ff99caf..cd6a3894 100644 --- a/mart/configs/experiment/COCO_TorchvisionFasterRCNN_Adv.yaml +++ b/mart/configs/experiment/COCO_TorchvisionFasterRCNN_Adv.yaml @@ -3,6 +3,7 @@ defaults: - COCO_TorchvisionFasterRCNN - /attack@callbacks.adversary_connector.test_adversary: object_detection_mask_adversary + - override /batch_c15n@callbacks.adversary_connector.batch_c15n: input_tuple_float01 - override /datamodule: coco_perturbable_mask - override /callbacks: [model_checkpoint, lr_monitor, adversary_connector] diff --git a/mart/configs/model/classifier_cifar10_cnn.yaml b/mart/configs/model/classifier_cifar10_cnn.yaml index ba7455b1..f31ee799 100644 --- a/mart/configs/model/classifier_cifar10_cnn.yaml +++ b/mart/configs/model/classifier_cifar10_cnn.yaml @@ -4,7 +4,7 @@ defaults: modules: preprocessor: - # Normalize [0, 255] input. + # Normalize [0, 1] input. _target_: torchvision.transforms.Normalize - mean: [125.307, 122.961, 113.8575] - std: [51.5865, 50.847, 51.255] + mean: [0.4914, 0.4822, 0.4465] + std: [0.2023, 0.1994, 0.2010] diff --git a/mart/configs/model/classifier_timm.yaml b/mart/configs/model/classifier_timm.yaml index 0fa5ba1e..85f51e64 100644 --- a/mart/configs/model/classifier_timm.yaml +++ b/mart/configs/model/classifier_timm.yaml @@ -3,10 +3,11 @@ defaults: modules: preprocessor: - # Convert [0, 255] input to [0, 1] + # Use normalization parameters from timm + # https://github.com/huggingface/pytorch-image-models/blob/3d05c0e86f2f4c57bf495468aa3f8a7d3487c986/timm/data/constants.py#L3 _target_: torchvision.transforms.Normalize - mean: 0 - std: 255 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] logits: _target_: timm.models.convnext.convnext_tiny diff --git a/mart/configs/model/torchvision_faster_rcnn.yaml b/mart/configs/model/torchvision_faster_rcnn.yaml index 65200579..7e488183 100644 --- a/mart/configs/model/torchvision_faster_rcnn.yaml +++ b/mart/configs/model/torchvision_faster_rcnn.yaml @@ -11,12 +11,9 @@ training_step_log: training_sequence: seq010: - preprocessor: ["input"] + losses_and_detections: ["input", "target"] seq020: - losses_and_detections: ["preprocessor", "target"] - - seq030: loss: # Sum up the losses. [ @@ -28,17 +25,11 @@ training_sequence: validation_sequence: seq010: - preprocessor: ["input"] - - seq020: - losses_and_detections: ["preprocessor", "target"] + losses_and_detections: ["input", "target"] test_sequence: seq010: - preprocessor: ["input"] - - seq020: - losses_and_detections: ["preprocessor", "target"] + losses_and_detections: ["input", "target"] modules: losses_and_detections: diff --git a/mart/configs/model/torchvision_object_detection.yaml b/mart/configs/model/torchvision_object_detection.yaml index 534f0fc9..eb04b648 100644 --- a/mart/configs/model/torchvision_object_detection.yaml +++ b/mart/configs/model/torchvision_object_detection.yaml @@ -12,13 +12,6 @@ test_sequence: ??? output_preds_key: "losses_and_detections.eval" modules: - preprocessor: - _target_: mart.transforms.TupleTransforms - transforms: - _target_: torchvision.transforms.Normalize - mean: 0 - std: 255 - losses_and_detections: # Return losses in the training mode and predictions in the eval mode in one pass. _target_: mart.models.DualMode diff --git a/mart/configs/model/torchvision_retinanet.yaml b/mart/configs/model/torchvision_retinanet.yaml index 34b66945..a261a0e2 100644 --- a/mart/configs/model/torchvision_retinanet.yaml +++ b/mart/configs/model/torchvision_retinanet.yaml @@ -8,8 +8,7 @@ training_step_log: loss_box_reg: "losses_and_detections.training.bbox_regression" training_sequence: - - preprocessor: ["input"] - - losses_and_detections: ["preprocessor", "target"] + - losses_and_detections: ["input", "target"] - loss: # Sum up the losses. [ @@ -18,12 +17,10 @@ training_sequence: ] validation_sequence: - - preprocessor: ["input"] - - losses_and_detections: ["preprocessor", "target"] + - losses_and_detections: ["input", "target"] test_sequence: - - preprocessor: ["input"] - - losses_and_detections: ["preprocessor", "target"] + - losses_and_detections: ["input", "target"] modules: losses_and_detections: diff --git a/tests/conftest.py b/tests/conftest.py index 220f8734..0eaf4812 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def cfg_experiment(request) -> DictConfig: @pytest.fixture(scope="function") def input_data(): image_size = (3, 32, 32) - return torch.randint(0, 256, image_size, dtype=torch.float) + return torch.rand(image_size, dtype=torch.float) @pytest.fixture(scope="function") @@ -59,5 +59,5 @@ def target_data(): @pytest.fixture(scope="function") def perturbation(): torch.manual_seed(0) - perturbation = torch.randint(0, 256, (3, 32, 32), dtype=torch.float) + perturbation = torch.rand((3, 32, 32), dtype=torch.float) return perturbation diff --git a/tests/test_adversary.py b/tests/test_adversary.py index 15dc6e28..4c939506 100644 --- a/tests/test_adversary.py +++ b/tests/test_adversary.py @@ -211,7 +211,12 @@ def model(input, target): perturbation = input_data - input_adv - torch.testing.assert_close(perturbation.unique(), torch.Tensor([-1, 0, 1])) + # torch.unique() does not approximate floating number with threshold. So do it manually with 1e-6. + expansion = torch.tensor([19.9316]) # 1 / (2 ** 19.9316) = 1e-6 + unique_int = perturbation.ldexp(expansion).round().unique() + unique = unique_int.ldexp(-expansion) + + torch.testing.assert_close(unique, torch.Tensor([-1, 0, 1])) def test_configure_optimizers(): diff --git a/tests/test_visualizer.py b/tests/test_visualizer.py index d4788e7d..a0cf6bce 100644 --- a/tests/test_visualizer.py +++ b/tests/test_visualizer.py @@ -15,12 +15,15 @@ def test_visualizer_run_end(input_data, target_data, perturbation, tmp_path): folder = tmp_path / "test" - input_list = [input_data] + + # Convert to [0,255] space for adversary. + perturbation_255 = perturbation * 255 + input_list = [input_data * 255] target_list = [target_data] # simulate an addition perturbation def perturb(input, target): - result = [sample + perturbation for sample in input] + result = [sample + perturbation_255 for sample in input] return result, target adversary = Mock(spec=Adversary, side_effect=perturb) @@ -42,7 +45,7 @@ def perturb(input, target): # verify image file content perturbed_img = input_data + perturbation converter = ToPILImage() - expected_img = converter(perturbed_img / 255) + expected_img = converter(perturbed_img) expected_img.save(folder / "test_expected.jpg") stored_img = Image.open(expected_output_path)