Merge branch 'main' into feat-374142081-add-source-column-match

Author: chalmerlowe

google/cloud/bigquery/external_config.py

@@ -508,6 +508,27 @@ def source_column_match(self, value: Optional[SourceColumnMatch]):
             )
         self._properties["sourceColumnMatch"] = value.value if value else None
 
+    @property
+    def null_markers(self) -> Optional[Iterable[str]]:
+        """Optional[Iterable[str]]: A list of strings represented as SQL NULL values in a CSV file.
+
+        .. note::
+            null_marker and null_markers can't be set at the same time.
+            If null_marker is set, null_markers has to be not set.
+            If null_markers is set, null_marker has to be not set.
+            If both null_marker and null_markers are set at the same time, a user error would be thrown.
+            Any strings listed in null_markers, including empty string would be interpreted as SQL NULL.
+            This applies to all column types.
+
+        See
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.null_markers
+        """
+        return self._properties.get("nullMarkers")
+
+    @null_markers.setter
+    def null_markers(self, value: Optional[Iterable[str]]):
+        self._properties["nullMarkers"] = value
+
     def to_api_repr(self) -> dict:
         """Build an API representation of this object.
 

google/cloud/bigquery/job/load.py

@@ -387,6 +387,27 @@ def null_marker(self):
     def null_marker(self, value):
         self._set_sub_prop("nullMarker", value)
 
+    @property
+    def null_markers(self) -> Optional[List[str]]:
+        """Optional[List[str]]: A list of strings represented as SQL NULL values in a CSV file.
+
+        .. note::
+            null_marker and null_markers can't be set at the same time.
+            If null_marker is set, null_markers has to be not set.
+            If null_markers is set, null_marker has to be not set.
+            If both null_marker and null_markers are set at the same time, a user error would be thrown.
+            Any strings listed in null_markers, including empty string would be interpreted as SQL NULL.
+            This applies to all column types.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_markers
+        """
+        return self._get_sub_prop("nullMarkers")
+
+    @null_markers.setter
+    def null_markers(self, value: Optional[List[str]]):
+        self._set_sub_prop("nullMarkers", value)
+
     @property
     def preserve_ascii_control_characters(self):
         """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.
@@ -888,6 +909,13 @@ def null_marker(self):
         """
         return self.configuration.null_marker
 
+    @property
+    def null_markers(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.null_markers`.
+        """
+        return self.configuration.null_markers
+
     @property
     def quote_character(self):
         """See

tests/unit/job/test_load.py

@@ -44,6 +44,7 @@ def _setUpConstants(self):
         self.TIME_ZONE = "UTC"
         self.TIME_FORMAT = "%H:%M:%S"
         self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
+        self.NULL_MARKERS = ["", "NA"]
 
     def _make_resource(self, started=False, ended=False):
         resource = super(TestLoadJob, self)._make_resource(started, ended)
@@ -55,6 +56,7 @@ def _make_resource(self, started=False, ended=False):
         config["timeZone"] = self.TIME_ZONE
         config["timeFormat"] = self.TIME_FORMAT
         config["timestampFormat"] = self.TIMESTAMP_FORMAT
+        config["nullMarkers"] = self.NULL_MARKERS
 
         config["destinationTable"] = {
             "projectId": self.PROJECT,
@@ -143,6 +145,10 @@ def _verifyResourceProperties(self, job, resource):
             self.assertEqual(job.null_marker, config["nullMarker"])
         else:
             self.assertIsNone(job.null_marker)
+        if "nullMarkers" in config:
+            self.assertEqual(job.null_markers, config["nullMarkers"])
+        else:
+            self.assertIsNone(job.null_markers)
         if "quote" in config:
             self.assertEqual(job.quote_character, config["quote"])
         else:
@@ -223,6 +229,7 @@ def test_ctor(self):
         self.assertIsNone(job.ignore_unknown_values)
         self.assertIsNone(job.max_bad_records)
         self.assertIsNone(job.null_marker)
+        self.assertIsNone(job.null_markers)
         self.assertIsNone(job.quote_character)
         self.assertIsNone(job.skip_leading_rows)
         self.assertIsNone(job.source_format)

tests/unit/job/test_load_config.py

@@ -469,6 +469,22 @@ def test_null_marker_setter(self):
         config.null_marker = null_marker
         self.assertEqual(config._properties["load"]["nullMarker"], null_marker)
 
+    def test_null_markers_missing(self):
+        config = self._get_target_class()()
+        self.assertIsNone(config.null_markers)
+
+    def test_null_markers_hit(self):
+        null_markers = ["", "NA"]
+        config = self._get_target_class()()
+        config._properties["load"]["nullMarkers"] = null_markers
+        self.assertEqual(config.null_markers, null_markers)
+
+    def test_null_markers_setter(self):
+        null_markers = ["", "NA"]
+        config = self._get_target_class()()
+        config.null_markers = null_markers
+        self.assertEqual(config._properties["load"]["nullMarkers"], null_markers)
+
     def test_preserve_ascii_control_characters_missing(self):
         config = self._get_target_class()()
         self.assertIsNone(config.preserve_ascii_control_characters)

tests/unit/test_external_config.py

@@ -280,6 +280,7 @@ def test_from_api_repr_csv(self):
                     "encoding": "encoding",
                     "preserveAsciiControlCharacters": False,
                     "sourceColumnMatch": self.SOURCE_COLUMN_MATCH,
+                    "nullMarkers": ["", "NA"],
                 },
             },
         )
@@ -300,6 +301,7 @@ def test_from_api_repr_csv(self):
             ec.options.source_column_match,
             self.SOURCE_COLUMN_MATCH,
         )
+        self.assertEqual(ec.options.null_markers, ["", "NA"])
 
         got_resource = ec.to_api_repr()
 
@@ -322,6 +324,8 @@ def test_to_api_repr_csv(self):
         options.allow_jagged_rows = False
         options.preserve_ascii_control_characters = False
         options.source_column_match = self.SOURCE_COLUMN_MATCH
+        options.null_markers = ["", "NA"]
+
         ec.csv_options = options
 
         exp_resource = {
@@ -335,6 +339,7 @@ def test_to_api_repr_csv(self):
                 "encoding": "encoding",
                 "preserveAsciiControlCharacters": False,
                 "sourceColumnMatch": self.SOURCE_COLUMN_MATCH,
+                "nullMarkers": ["", "NA"],
             },
         }