From 5a2816fa2954774fdda130cb5ac7199967f2c363 Mon Sep 17 00:00:00 2001 From: pdamasceno Date: Sun, 31 May 2026 12:24:59 +0200 Subject: [PATCH 1/3] Add file adapter for syncing from local CSV exports --- docs/docs/adapters/file.mdx | 145 ++++++++++ docs/sidebars.ts | 1 + examples/file_to_infrahub/config.yml | 49 ++++ examples/file_to_infrahub/data/devices.csv | 4 + .../file_to_infrahub/data/organizations.csv | 3 + infrahub_sync/adapters/file.py | 249 ++++++++++++++++++ tests/adapters/test_file.py | 153 +++++++++++ 7 files changed, 604 insertions(+) create mode 100644 docs/docs/adapters/file.mdx create mode 100644 examples/file_to_infrahub/config.yml create mode 100644 examples/file_to_infrahub/data/devices.csv create mode 100644 examples/file_to_infrahub/data/organizations.csv create mode 100644 infrahub_sync/adapters/file.py create mode 100644 tests/adapters/test_file.py diff --git a/docs/docs/adapters/file.mdx b/docs/docs/adapters/file.mdx new file mode 100644 index 0000000..e1ce0da --- /dev/null +++ b/docs/docs/adapters/file.mdx @@ -0,0 +1,145 @@ +--- +title: File adapter +--- + +import ReferenceLink from "../../src/components/Card"; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +The File adapter loads data from local file exports and synchronizes it into Infrahub. It is designed for the common case where another system can export flat files (CSV today) that you want to ingest without standing up an API integration. + +The adapter is built around a small, pluggable reader registry, so support for additional formats (JSON, YAML, Excel) can be added without changing the adapter logic. + +## Key features + +- Reads local CSV exports into Infrahub models +- File format inferred from the file extension, with an optional `format` override +- Configurable base directory, delimiter, and encoding +- Filtering and transformation support (shared with the other adapters) +- Scalar and list references to other models by identifier + +## Sync directions supported + +- File → Infrahub + +:::info + +The File adapter currently supports only **one-way synchronization** from local files into Infrahub. +Writing data back to files is not yet supported. + +::: + +## Supported formats + +| Format | Extension | Status | +|--------|-----------|--------| +| CSV | `.csv` | Supported | +| JSON | `.json` | Planned | +| YAML | `.yml`, `.yaml` | Planned | +| Excel | `.xlsx` | Planned | + +If a file uses an unrecognized extension, set the `format` setting explicitly. An unsupported format raises a clear error listing the formats that are available. + +## Configuration + +### Basic configuration + +To use the File adapter, specify `file` as the source name and point the schema mapping at your exported files: + +```yaml +source: + name: file + settings: + directory: "examples/file_to_infrahub/data" + delimiter: "," + encoding: "utf-8" +``` + +### Configuration parameters + +| Parameter | Description | Default | Required | +|-----------|-------------|---------|----------| +| `directory` | Base directory used to resolve relative file paths | Config directory, then CWD | No | +| `format` | Force a format instead of inferring from the extension (for example, `csv`) | Inferred from extension | No | +| `delimiter` | CSV field separator | `,` | No | +| `encoding` | File encoding | `utf-8` | No | +| `empty_as_none` | Treat empty CSV cells as `null` instead of `""` | `true` | No | +| `list_separator` | Separator used to split list/reference fields | `;` | No | + +## Schema mapping + +The `mapping` key of each schema mapping entry is the file to read, interpreted relative to the configured `directory` (or as an absolute path): + +```yaml +order: + - OrganizationGeneric + - InfraDevice + +schema_mapping: + - name: OrganizationGeneric + mapping: organizations.csv + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: description + mapping: description + + - name: InfraDevice + mapping: devices.csv + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: organization + mapping: organization + reference: OrganizationGeneric +``` + +### Field mapping types + +- **Direct mapping**: Maps a column to an Infrahub field. +- **Static value**: Sets a constant value for an Infrahub field. +- **Reference**: Links to another Infrahub model. For flat files the referenced column holds the target object's identifier (for example, the organization name). + +### List and reference values + +A column that maps to a list attribute (or a list reference) is split using `list_separator` (default `;`). For example, a `tags` column with the value `core;edge` becomes `["core", "edge"]`. + +### Identifiers + +Files rarely carry a database `id`. The adapter derives a source-local identifier in this order: + +1. An `id` column, or any `*_id` column with a value. +2. The configured `identifiers` joined together. +3. The row index as a last resort. + +## Example data + +Given `organizations.csv`: + +```csv +name,description +Acme,Acme headquarters +Globex,Globex regional office +``` + +and `devices.csv`: + +```csv +name,description,organization +device-001,Core router,Acme +device-002,Access switch,Acme +``` + +the `organization` column on each device resolves to the matching `OrganizationGeneric` object. + +## Common errors + +| Error | Cause | Resolution | +|-------|-------|------------| +| `File not found: ...` | The mapped file does not exist relative to `directory` | Check the `directory` setting and the `mapping` path | +| `Unsupported file format for ...` | The extension is not recognized and no `format` is set | Use a `.csv` file or set the `format` setting | +| `Error reading data from ...` | The file could not be parsed (for example, malformed CSV) | Verify the delimiter, encoding, and file contents | + + diff --git a/docs/sidebars.ts b/docs/sidebars.ts index bf159cc..a60ecb2 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -17,6 +17,7 @@ const sidebars: SidebarsConfig = { label: 'Adapters', items: [ 'adapters/aci', + 'adapters/file', 'adapters/genericrestapi', 'adapters/infrahub', 'adapters/ipfabric', diff --git a/examples/file_to_infrahub/config.yml b/examples/file_to_infrahub/config.yml new file mode 100644 index 0000000..260423c --- /dev/null +++ b/examples/file_to_infrahub/config.yml @@ -0,0 +1,49 @@ +--- +name: from-file + +source: + name: file + settings: + # Base directory used to resolve relative file paths below. + # Defaults to this config's directory when omitted. + directory: "examples/file_to_infrahub/data" + # CSV options (applied to all .csv files) + delimiter: "," + encoding: "utf-8" + +destination: + name: infrahub + settings: + url: "http://localhost:8000" + +# Skip objects in the source that don't exist in the destination (prevents creation) +diffsync_flags: ["SKIP_UNMATCHED_SRC"] + +order: [ + "OrganizationGeneric", + "InfraDevice", +] + +schema_mapping: + # Sites/organizations exported to organizations.csv + - name: OrganizationGeneric + mapping: organizations.csv + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: description + mapping: description + + # Devices exported to devices.csv, referencing an organization by name + - name: InfraDevice + mapping: devices.csv + identifiers: ["name"] + fields: + - name: name + mapping: name + - name: description + mapping: description + - name: organization + mapping: organization + reference: OrganizationGeneric diff --git a/examples/file_to_infrahub/data/devices.csv b/examples/file_to_infrahub/data/devices.csv new file mode 100644 index 0000000..264ec33 --- /dev/null +++ b/examples/file_to_infrahub/data/devices.csv @@ -0,0 +1,4 @@ +name,description,organization +device-001,Core router,Acme +device-002,Access switch,Acme +device-003,Edge firewall,Globex diff --git a/examples/file_to_infrahub/data/organizations.csv b/examples/file_to_infrahub/data/organizations.csv new file mode 100644 index 0000000..1ba6f68 --- /dev/null +++ b/examples/file_to_infrahub/data/organizations.csv @@ -0,0 +1,3 @@ +name,description +Acme,Acme headquarters +Globex,Globex regional office diff --git a/infrahub_sync/adapters/file.py b/infrahub_sync/adapters/file.py new file mode 100644 index 0000000..97adada --- /dev/null +++ b/infrahub_sync/adapters/file.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import csv +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from diffsync import Adapter, DiffSyncModel +from typing_extensions import Self + +from infrahub_sync import ( + DiffSyncMixin, + DiffSyncModelMixin, + SchemaMappingModel, + SyncAdapter, + SyncConfig, +) + +from .utils import get_value + +if TYPE_CHECKING: + from collections.abc import Callable + +logger = logging.getLogger(__name__) + + +def read_csv(path: Path, settings: dict[str, Any]) -> list[dict[str, Any]]: + """Read a CSV file into a list of dictionaries keyed by the header row. + + Recognized settings: + delimiter: Field separator (default ","). + encoding: File encoding (default "utf-8"). + empty_as_none: Treat empty cells as ``None`` instead of "" (default True). + """ + delimiter = settings.get("delimiter", ",") + encoding = settings.get("encoding", "utf-8") + empty_as_none = settings.get("empty_as_none", True) + + records: list[dict[str, Any]] = [] + with path.open(newline="", encoding=encoding) as handle: + reader = csv.DictReader(handle, delimiter=delimiter) + if reader.fieldnames is None: + return records + for row in reader: + if empty_as_none: + records.append({key: (value or None) for key, value in row.items()}) + else: + records.append(dict(row)) + return records + + +# Registry of file readers keyed by normalized format name. Adding support for a +# new format (json, yaml, excel, ...) is a matter of writing a reader function +# with the same signature and registering it here. +FILE_READERS: dict[str, Callable[[Path, dict[str, Any]], list[dict[str, Any]]]] = { + "csv": read_csv, +} + +# Map file extensions to a format name in FILE_READERS. +EXTENSION_FORMATS: dict[str, str] = { + ".csv": "csv", +} + + +class FileAdapter(DiffSyncMixin, Adapter): + """Adapter that loads data from local file exports (CSV today, extensible to other formats). + + Each entry in ``schema_mapping`` maps a model to a file via its ``mapping`` key, + which is interpreted as a path relative to the configured base directory (or an + absolute path). The file format is inferred from the extension unless overridden + through the ``format`` setting. + """ + + type = "File" + + def __init__(self, target: str, adapter: SyncAdapter, config: SyncConfig, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + self.target = target + self.settings = adapter.settings or {} + self.config = config + self.base_directory = self._resolve_base_directory(config=config) + self.format_override = self.settings.get("format") + + def _resolve_base_directory(self, config: SyncConfig) -> Path: + """Determine the directory used to resolve relative file paths. + + Priority: the ``directory`` setting, then the sync instance directory, then CWD. + """ + directory = self.settings.get("directory") + if directory: + return Path(directory) + instance_directory = getattr(config, "directory", None) + if instance_directory: + return Path(instance_directory) + return Path.cwd() + + def _resolve_path(self, mapping: str) -> Path: + """Resolve a schema mapping value to an absolute file path.""" + path = Path(mapping) + if not path.is_absolute(): + path = self.base_directory / path + return path + + def _read_file(self, path: Path) -> list[dict[str, Any]]: + """Read a file using the reader registered for its format.""" + if not path.is_file(): + msg = f"File not found: {path}" + raise FileNotFoundError(msg) + + file_format = (self.format_override or EXTENSION_FORMATS.get(path.suffix.lower()) or "").lower() + reader = FILE_READERS.get(file_format) + if reader is None: + supported = ", ".join(sorted(FILE_READERS)) or "none" + msg = ( + f"Unsupported file format for '{path}'. " + f"Supported formats: {supported}. " + f"Set the 'format' setting explicitly or use a recognized extension " + f"({', '.join(sorted(EXTENSION_FORMATS))})." + ) + raise ValueError(msg) + + return reader(path, self.settings) + + def model_loader(self, model_name: str, model: type[FileModel]) -> None: + """Load and process models using schema mapping filters and transformations. + + This reads each mapped file, applies filters and transformations as specified + in the schema mapping, and loads the processed records into the adapter. + """ + for element in self.config.schema_mapping: + if element.name != model_name: + continue + + if not element.mapping: + logger.info("No mapping defined for '%s', skipping", element.name) + continue + + path = self._resolve_path(mapping=element.mapping) + try: + objs = self._read_file(path=path) + except (FileNotFoundError, ValueError, OSError, csv.Error) as exc: + msg = f"Error reading data from '{path}': {exc!s}" + raise ValueError(msg) from exc + + total = len(objs) + adapter_type_title = (self.type or "").title() + if self.config.source.name.title() == adapter_type_title: + # Filter records + filtered_objs = model.filter_records(records=objs, schema_mapping=element) + logger.info("%s: Loading %d/%d %s", self.type, len(filtered_objs), total, element.mapping) + # Transform records + transformed_objs = model.transform_records(records=filtered_objs, schema_mapping=element) + else: + logger.info("%s: Loading all %d %s", self.type, total, element.mapping) + transformed_objs = objs + + # Create model instances after filtering and transforming + for index, obj in enumerate(transformed_objs): + data = self.obj_to_diffsync(obj=obj, mapping=element, model=model, index=index) + item = model(**data) + self.add(item) + + def _derive_local_id(self, obj: dict[str, Any], mapping: SchemaMappingModel, index: int) -> str: + """Derive a source-local identifier for a record. + + Files rarely carry a database ``id``, so fall back to ``id``/``*_id`` columns, + then the configured identifiers, then the row index. + """ + obj_id = obj.get("id") + if obj_id is None: + for key, value in obj.items(): + if key.endswith("_id") and value: + obj_id = value + break + if obj_id is None and mapping.identifiers: + parts = [str(obj.get(identifier, "")) for identifier in mapping.identifiers] + if any(parts): + obj_id = "__".join(parts) + if obj_id is None: + obj_id = str(index) + return str(obj_id) + + def obj_to_diffsync( + self, obj: dict[str, Any], mapping: SchemaMappingModel, model: type[FileModel], index: int + ) -> dict[str, Any]: + """Convert a file record into a DiffSync-ready dictionary. + + Supports static values, simple field mappings, and scalar/list references that + point at another model's identifier (the natural shape for flat file exports). + """ + data: dict[str, Any] = {"local_id": self._derive_local_id(obj=obj, mapping=mapping, index=index)} + + if not mapping.fields: + return data + + list_separator = self.settings.get("list_separator", ";") + + for field in mapping.fields: + field_is_list = model.is_list(name=field.name) + + if field.static: + data[field.name] = field.static + elif not field_is_list and field.mapping and not field.reference: + value = get_value(obj, field.mapping) + if value is not None: + data[field.name] = value + elif field_is_list and field.mapping and not field.reference: + value = get_value(obj, field.mapping) + if value is not None: + data[field.name] = self._split_list(value=value, separator=list_separator) + elif field.mapping and field.reference: + value = get_value(obj, field.mapping) + if not field_is_list: + if value is not None: + # Flat files reference another object directly by its identifier. + data[field.name] = value + else: + references = self._split_list(value=value, separator=list_separator) if value is not None else [] + data[field.name] = sorted(str(item) for item in references if item not in (None, "")) + + return data + + @staticmethod + def _split_list(value: Any, separator: str) -> list[Any]: + """Normalize a value into a list, splitting delimited strings.""" + if isinstance(value, list): + return value + if isinstance(value, str): + return [part.strip() for part in value.split(separator) if part.strip()] + return [value] + + +class FileModel(DiffSyncModelMixin, DiffSyncModel): + """Model for the file adapter. The file adapter is read-only (source) for now.""" + + @classmethod + def create( + cls, + adapter: Adapter, + ids: dict[Any, Any], + attrs: dict[Any, Any], + ) -> Self | None: + # The file adapter is a source-only adapter; writing back to files is not supported yet. + return super().create(adapter=adapter, ids=ids, attrs=attrs) + + def update(self, attrs: dict) -> Self | None: + # The file adapter is a source-only adapter; writing back to files is not supported yet. + return super().update(attrs=attrs) diff --git a/tests/adapters/test_file.py b/tests/adapters/test_file.py new file mode 100644 index 0000000..36f9fcd --- /dev/null +++ b/tests/adapters/test_file.py @@ -0,0 +1,153 @@ +"""Unit tests for the file adapter (CSV reader, identifier derivation, references, errors).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import pytest +from diffsync.store.local import LocalStore + +from infrahub_sync import SyncInstance +from infrahub_sync.adapters.file import FileAdapter, FileModel, read_csv + +if TYPE_CHECKING: + from pathlib import Path + + +class OrganizationGeneric(FileModel): + _modelname = "OrganizationGeneric" + _identifiers = ("name",) + _attributes = ("description",) + name: str + description: str | None = None + + +class InfraDevice(FileModel): + _modelname = "InfraDevice" + _identifiers = ("name",) + _attributes = ("description", "organization", "tags") + name: str + description: str | None = None + organization: str | None = None + tags: list[str] = [] # noqa: RUF012 + + +class _FileSync(FileAdapter): + top_level = ["OrganizationGeneric", "InfraDevice"] # noqa: RUF012 + OrganizationGeneric = OrganizationGeneric + InfraDevice = InfraDevice + + +def _build_instance(directory: Path) -> SyncInstance: + """Build a minimal SyncInstance for the file adapter pointing at a data directory.""" + return SyncInstance.model_validate( + { + "name": "from-file-test", + "directory": str(directory), + "source": {"name": "file", "settings": {"directory": str(directory)}}, + "destination": {"name": "infrahub", "settings": {}}, + "order": ["OrganizationGeneric", "InfraDevice"], + "schema_mapping": [ + { + "name": "OrganizationGeneric", + "mapping": "organizations.csv", + "identifiers": ["name"], + "fields": [ + {"name": "name", "mapping": "name"}, + {"name": "description", "mapping": "description"}, + ], + }, + { + "name": "InfraDevice", + "mapping": "devices.csv", + "identifiers": ["name"], + "fields": [ + {"name": "name", "mapping": "name"}, + {"name": "description", "mapping": "description"}, + {"name": "organization", "mapping": "organization", "reference": "OrganizationGeneric"}, + {"name": "tags", "mapping": "tags"}, + ], + }, + ], + } + ) + + +@pytest.fixture +def data_dir(tmp_path: Path) -> Path: + """Write sample CSV files and return the directory containing them.""" + (tmp_path / "organizations.csv").write_text("name,description\nAcme,HQ\nGlobex,\n", encoding="utf-8") + (tmp_path / "devices.csv").write_text( + "name,description,organization,tags\ndevice-1,Router,Acme,core;edge\n", + encoding="utf-8", + ) + return tmp_path + + +def test_read_csv_empty_cell_becomes_none(tmp_path: Path) -> None: + """An empty CSV cell is converted to None by default.""" + path = tmp_path / "orgs.csv" + path.write_text("name,description\nGlobex,\n", encoding="utf-8") + records = read_csv(path, settings={}) + assert records == [{"name": "Globex", "description": None}] + + +def test_read_csv_respects_delimiter(tmp_path: Path) -> None: + """A custom delimiter is honored when reading.""" + path = tmp_path / "orgs.csv" + path.write_text("name;description\nAcme;HQ\n", encoding="utf-8") + records = read_csv(path, settings={"delimiter": ";"}) + assert records == [{"name": "Acme", "description": "HQ"}] + + +def test_model_loader_loads_records_and_resolves_reference(data_dir: Path) -> None: + """Records load, identifiers seed local_id, and a scalar reference is kept as the target identifier.""" + instance = _build_instance(directory=data_dir) + adapter = _FileSync(target="source", adapter=instance.source, config=instance, internal_storage_engine=LocalStore()) + adapter.load() + + orgs = sorted(cast("list[OrganizationGeneric]", adapter.get_all("OrganizationGeneric")), key=lambda o: o.name) + assert [o.name for o in orgs] == ["Acme", "Globex"] + assert orgs[0].local_id == "Acme" + + device = cast("InfraDevice", adapter.get_all("InfraDevice")[0]) + assert device.organization == "Acme" + assert device.tags == ["core", "edge"] + + +def _single_model_instance(directory: Path, mapping: str) -> SyncInstance: + """Build a SyncInstance with a single InfraDevice model mapped to ``mapping``.""" + return SyncInstance.model_validate( + { + "name": "from-file-test", + "directory": str(directory), + "source": {"name": "file", "settings": {"directory": str(directory)}}, + "destination": {"name": "infrahub", "settings": {}}, + "order": ["InfraDevice"], + "schema_mapping": [ + { + "name": "InfraDevice", + "mapping": mapping, + "identifiers": ["name"], + "fields": [{"name": "name", "mapping": "name"}], + } + ], + } + ) + + +def test_unsupported_format_raises(tmp_path: Path) -> None: + """An unrecognized extension yields a clear error listing supported formats.""" + (tmp_path / "devices.txt").write_text("name\ndevice-1\n", encoding="utf-8") + instance = _single_model_instance(directory=tmp_path, mapping="devices.txt") + adapter = _FileSync(target="source", adapter=instance.source, config=instance, internal_storage_engine=LocalStore()) + with pytest.raises(ValueError, match="Unsupported file format"): + adapter.model_loader(model_name="InfraDevice", model=InfraDevice) + + +def test_missing_file_raises(tmp_path: Path) -> None: + """A missing mapped file raises a descriptive error.""" + instance = _single_model_instance(directory=tmp_path, mapping="missing.csv") + adapter = _FileSync(target="source", adapter=instance.source, config=instance, internal_storage_engine=LocalStore()) + with pytest.raises(ValueError, match="Error reading data"): + adapter.model_loader(model_name="InfraDevice", model=InfraDevice) From 51c962774f5fb07787469ece7d4ddc180a9444c6 Mon Sep 17 00:00:00 2001 From: pdamasceno Date: Sun, 31 May 2026 14:51:26 +0200 Subject: [PATCH 2/3] - Add parametrized empty-CSV test (fully-empty and header-only) - Simplify example config: drop repo-root-bound directory, redundant delimiter/encoding defaults; prefix mappings with data/ --- examples/file_to_infrahub/config.yml | 12 +++--------- tests/adapters/test_file.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/examples/file_to_infrahub/config.yml b/examples/file_to_infrahub/config.yml index 260423c..a2a87fa 100644 --- a/examples/file_to_infrahub/config.yml +++ b/examples/file_to_infrahub/config.yml @@ -3,13 +3,7 @@ name: from-file source: name: file - settings: - # Base directory used to resolve relative file paths below. - # Defaults to this config's directory when omitted. - directory: "examples/file_to_infrahub/data" - # CSV options (applied to all .csv files) - delimiter: "," - encoding: "utf-8" + settings: {} destination: name: infrahub @@ -27,7 +21,7 @@ order: [ schema_mapping: # Sites/organizations exported to organizations.csv - name: OrganizationGeneric - mapping: organizations.csv + mapping: data/organizations.csv identifiers: ["name"] fields: - name: name @@ -37,7 +31,7 @@ schema_mapping: # Devices exported to devices.csv, referencing an organization by name - name: InfraDevice - mapping: devices.csv + mapping: data/devices.csv identifiers: ["name"] fields: - name: name diff --git a/tests/adapters/test_file.py b/tests/adapters/test_file.py index 36f9fcd..f8a7083 100644 --- a/tests/adapters/test_file.py +++ b/tests/adapters/test_file.py @@ -100,6 +100,17 @@ def test_read_csv_respects_delimiter(tmp_path: Path) -> None: assert records == [{"name": "Acme", "description": "HQ"}] +@pytest.mark.parametrize( + ("content", "case"), + [("", "fully-empty"), ("name,description\n", "header-only")], +) +def test_read_csv_empty_returns_no_records(tmp_path: Path, content: str, case: str) -> None: + """A CSV with no data rows yields an empty list whether or not a header is present.""" + path = tmp_path / f"orgs-{case}.csv" + path.write_text(content, encoding="utf-8") + assert read_csv(path, settings={}) == [] + + def test_model_loader_loads_records_and_resolves_reference(data_dir: Path) -> None: """Records load, identifiers seed local_id, and a scalar reference is kept as the target identifier.""" instance = _build_instance(directory=data_dir) From 6aa2d5f4fe23e65570b3d215dc053b50e59bde34 Mon Sep 17 00:00:00 2001 From: pdamasceno Date: Sun, 31 May 2026 15:17:53 +0200 Subject: [PATCH 3/3] bug/fix: Dedupe rows producing the same identifier in file adapter Flat-file sources naturally produce duplicate identifiers when a model is derived from a foreign-key column (e.g. one LocationSite per unique 'location' value across many device rows). Previously the second occurrence raised ObjectAlreadyExists and aborted the load. Now the first occurrence wins and a summary count is logged. --- infrahub_sync/adapters/file.py | 19 ++++++++++++++++-- tests/adapters/test_file.py | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/infrahub_sync/adapters/file.py b/infrahub_sync/adapters/file.py index 97adada..9371fa8 100644 --- a/infrahub_sync/adapters/file.py +++ b/infrahub_sync/adapters/file.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from diffsync import Adapter, DiffSyncModel +from diffsync.exceptions import ObjectAlreadyExists from typing_extensions import Self from infrahub_sync import ( @@ -155,11 +156,25 @@ def model_loader(self, model_name: str, model: type[FileModel]) -> None: logger.info("%s: Loading all %d %s", self.type, total, element.mapping) transformed_objs = objs - # Create model instances after filtering and transforming + # Flat-file rows often repeat the same identifier when a model is derived from a + # foreign-key column (e.g. every device row carries the same site name). + # Keep the first occurrence and skip subsequent duplicates so the load completes. + duplicates = 0 for index, obj in enumerate(transformed_objs): data = self.obj_to_diffsync(obj=obj, mapping=element, model=model, index=index) item = model(**data) - self.add(item) + try: + self.add(item) + except ObjectAlreadyExists: + duplicates += 1 + if duplicates: + logger.info( + "%s: skipped %d duplicate %s in %s (first occurrence kept)", + self.type, + duplicates, + model_name, + element.mapping, + ) def _derive_local_id(self, obj: dict[str, Any], mapping: SchemaMappingModel, index: int) -> str: """Derive a source-local identifier for a record. diff --git a/tests/adapters/test_file.py b/tests/adapters/test_file.py index f8a7083..37d37a2 100644 --- a/tests/adapters/test_file.py +++ b/tests/adapters/test_file.py @@ -156,6 +156,41 @@ def test_unsupported_format_raises(tmp_path: Path) -> None: adapter.model_loader(model_name="InfraDevice", model=InfraDevice) +def test_duplicate_identifiers_are_deduped(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Multiple rows producing the same identifier load once instead of aborting the sync.""" + (tmp_path / "organizations.csv").write_text("name,description\nAcme,HQ\n", encoding="utf-8") + (tmp_path / "devices.csv").write_text( + "name,description,organization,tags\ndevice-1,Router,Acme,\ndevice-2,Switch,Acme,\ndevice-3,Firewall,Acme,\n", + encoding="utf-8", + ) + instance = SyncInstance.model_validate( + { + "name": "from-file-test", + "directory": str(tmp_path), + "source": {"name": "file", "settings": {"directory": str(tmp_path)}}, + "destination": {"name": "infrahub", "settings": {}}, + "order": ["OrganizationGeneric"], + "schema_mapping": [ + { + "name": "OrganizationGeneric", + "mapping": "devices.csv", + "identifiers": ["name"], + "fields": [ + {"name": "name", "mapping": "organization"}, + ], + }, + ], + } + ) + adapter = _FileSync(target="source", adapter=instance.source, config=instance, internal_storage_engine=LocalStore()) + with caplog.at_level("INFO", logger="infrahub_sync.adapters.file"): + adapter.load() + + orgs = adapter.get_all("OrganizationGeneric") + assert [o.get_unique_id() for o in orgs] == ["Acme"] + assert "skipped 2 duplicate OrganizationGeneric" in caplog.text + + def test_missing_file_raises(tmp_path: Path) -> None: """A missing mapped file raises a descriptive error.""" instance = _single_model_instance(directory=tmp_path, mapping="missing.csv")