diff --git a/cpp/src/arrow/csv/options.h b/cpp/src/arrow/csv/options.h index f0b923d0f323..6ddd6082103b 100644 --- a/cpp/src/arrow/csv/options.h +++ b/cpp/src/arrow/csv/options.h @@ -174,6 +174,16 @@ struct ARROW_EXPORT ReadOptions { Status Validate() const; }; +/// \brief Escape style for CSV writing +enum class ARROW_EXPORT EscapeStyle { + /// Quotes are escaped by doubling them, e.g. `""` (RFC4180 default) + Double, + /// Quotes are escaped by a preceding backslash, e.g. `\"` + Backslash, + /// Quotes are not escaped + None +}; + /// \brief Quoting style for CSV writing enum class ARROW_EXPORT QuotingStyle { /// Only enclose values in quotes which need them, because their CSV rendering can @@ -219,6 +229,14 @@ struct ARROW_EXPORT WriteOptions { /// effect of quoting all column names. QuotingStyle quoting_header = QuotingStyle::Needed; + /// \brief Escape style for quoting + /// + /// Controls how quotes within quoted values are escaped. + /// - `Double`: quotes are escaped by doubling them (e.g. `""`). RFC4180 default. + /// - `Backslash`: quotes are escaped by a preceding backslash (e.g. `\"`). + /// - `None`: quotes are not escaped. + EscapeStyle escape_style = EscapeStyle::Double; + /// Create write options with default values static WriteOptions Defaults(); diff --git a/cpp/src/arrow/csv/writer.cc b/cpp/src/arrow/csv/writer.cc index 2db0dba2de71..577ec85d2880 100644 --- a/cpp/src/arrow/csv/writer.cc +++ b/cpp/src/arrow/csv/writer.cc @@ -167,16 +167,32 @@ class ColumnPopulator { // Copies the contents of s to out properly escaping any necessary characters. // Returns the position next to last copied character. -char* Escape(std::string_view s, char* out) { +char* Escape(std::string_view s, char* out, EscapeStyle escape_style) { for (const char c : s) { + if (c == '"' && escape_style == EscapeStyle::Backslash) { + *out++ = '\\'; + } *out++ = c; - if (c == '"') { + if (c == '"' && escape_style == EscapeStyle::Double) { *out++ = '"'; } } return out; } +// Returns the number of characters needed to escape the given string. +int64_t EscapedLength(std::string_view s, EscapeStyle escape_style) { + int64_t quote_count = static_cast(std::count(s.begin(), s.end(), '"')); + switch (escape_style) { + case EscapeStyle::Double: + case EscapeStyle::Backslash: + return static_cast(s.length()) + quote_count; + case EscapeStyle::None: + return static_cast(s.length()); + } + return static_cast(s.length()); +} + // Return the index of the first structural char in the input. A structural char // is a character that needs quoting and/or escaping. int64_t StopAtStructuralChar(const uint8_t* data, const int64_t buffer_size, @@ -325,8 +341,10 @@ class UnquotedColumnPopulator : public ColumnPopulator { class QuotedColumnPopulator : public ColumnPopulator { public: QuotedColumnPopulator(MemoryPool* pool, std::string end_chars, - std::shared_ptr null_string) - : ColumnPopulator(pool, std::move(end_chars), std::move(null_string)) {} + std::shared_ptr null_string, + EscapeStyle escape_style) + : ColumnPopulator(pool, std::move(end_chars), std::move(null_string)), + escape_style_(escape_style) {} Status UpdateRowLengths(int64_t* row_lengths) override { if (ARROW_PREDICT_TRUE(array_->type_id() == Type::STRING)) { @@ -366,8 +384,7 @@ class QuotedColumnPopulator : public ColumnPopulator { // Each quote in the value string needs to be escaped. int64_t escaped_count = CountQuotes(s); row_needs_escaping_[row_number] = escaped_count > 0; - row_lengths[row_number] += - static_cast(s.length()) + escaped_count + kQuoteCount; + row_lengths[row_number] += EscapedLength(s, escape_style_) + kQuoteCount; row_number++; }, [&]() { @@ -401,7 +418,7 @@ class QuotedColumnPopulator : public ColumnPopulator { memcpy(row, s.data(), s.length()); row += s.length(); } else { - row = Escape(s, row); + row = Escape(s, row, escape_style_); } *row++ = '"'; CopyEndChars(row, end_chars_.data(), end_chars_.length()); @@ -436,12 +453,13 @@ class QuotedColumnPopulator : public ColumnPopulator { // at some point we should change this to use memory_pool // backed allocator. std::vector row_needs_escaping_; + const EscapeStyle escape_style_; }; Result> MakePopulator( const DataType& type, const std::string& end_chars, const char delimiter, const std::shared_ptr& null_string, QuotingStyle quoting_style, - MemoryPool* pool) { + EscapeStyle escape_style, MemoryPool* pool) { auto make_populator = [&](const auto& type) -> Result> { using Type = std::decay_t; @@ -459,7 +477,8 @@ Result> MakePopulator( pool, end_chars, delimiter, null_string, /*reject_values_with_quotes=*/false); case QuotingStyle::AllValid: - return std::make_unique(pool, end_chars, null_string); + return std::make_unique(pool, end_chars, null_string, + escape_style); } } @@ -479,13 +498,14 @@ Result> MakePopulator( case QuotingStyle::Needed: [[fallthrough]]; case QuotingStyle::AllValid: - return std::make_unique(pool, end_chars, null_string); + return std::make_unique(pool, end_chars, null_string, + escape_style); } } if constexpr (std::is_same::value) { return MakePopulator(*type.value_type(), end_chars, delimiter, null_string, - quoting_style, pool); + quoting_style, escape_style, pool); } return Status::Invalid("Unsupported Type:", type.ToString()); @@ -496,9 +516,9 @@ Result> MakePopulator( Result> MakePopulator( const Field& field, const std::string& end_chars, char delimiter, const std::shared_ptr& null_string, QuotingStyle quoting_style, - MemoryPool* pool) { + EscapeStyle escape_style, MemoryPool* pool) { return MakePopulator(*field.type(), end_chars, delimiter, null_string, quoting_style, - pool); + escape_style, pool); } class CSVWriterImpl : public ipc::RecordBatchWriter { @@ -525,7 +545,8 @@ class CSVWriterImpl : public ipc::RecordBatchWriter { ARROW_ASSIGN_OR_RAISE( populators[col], MakePopulator(*schema->field(col), end_chars, options.delimiter, null_string, - options.quoting_style, options.io_context.pool())); + options.quoting_style, options.escape_style, + options.io_context.pool())); } auto writer = std::make_shared( sink, std::move(owned_sink), std::move(schema), std::move(populators), options); @@ -601,13 +622,12 @@ class CSVWriterImpl : public ipc::RecordBatchWriter { int64_t header_length = 0; for (int col = 0; col < schema_->num_fields(); col++) { const std::string& col_name = schema_->field(col)->name(); - header_length += col_name.size(); + header_length += EscapedLength(col_name, options_.escape_style); switch (quoting_style) { case QuotingStyle::None: break; case QuotingStyle::Needed: case QuotingStyle::AllValid: - header_length += CountQuotes(col_name); break; } } @@ -649,7 +669,7 @@ class CSVWriterImpl : public ipc::RecordBatchWriter { // regardless of whether it contains structural chars. // We use consistent semantics for header names, which are strings. *next++ = '"'; - next = Escape(schema_->field(col)->name(), next); + next = Escape(schema_->field(col)->name(), next, options_.escape_style); *next++ = '"'; break; } diff --git a/cpp/src/arrow/csv/writer_test.cc b/cpp/src/arrow/csv/writer_test.cc index ce4d8ab16d01..67be6a2782cc 100644 --- a/cpp/src/arrow/csv/writer_test.cc +++ b/cpp/src/arrow/csv/writer_test.cc @@ -63,7 +63,8 @@ WriteOptions DefaultTestOptions(bool include_header = false, QuotingStyle quoting_style = QuotingStyle::Needed, const std::string& eol = "\n", char delimiter = ',', int batch_size = 5, - QuotingStyle quoting_header = QuotingStyle::Needed) { + QuotingStyle quoting_header = QuotingStyle::Needed, + EscapeStyle escape_style = EscapeStyle::Double) { WriteOptions options; options.batch_size = batch_size; options.include_header = include_header; @@ -72,6 +73,7 @@ WriteOptions DefaultTestOptions(bool include_header = false, options.eol = eol; options.quoting_style = quoting_style; options.delimiter = delimiter; + options.escape_style = escape_style; return options; } @@ -89,6 +91,21 @@ std::string UtilGetExpectedWithEOL(const std::string& eol) { R"(,"NA",,,,,,)" + eol; // line 11 } +// Expected output using EscapeStyle::Backslash (quotes escaped by a preceding backslash). +std::string UtilGetExpectedWithEOLBackslash(const std::string& eol) { + return std::string("1,,-1,,,,,") + eol + // line 1 + R"(1,"abc\"efg",2324,,,,,)" + eol + // line 2 + R"(,"abcd",5467,,,,,"efghi")" + eol + // line 3 + R"(,,,,,,,)" + eol + // line 4 + R"(546,"",517,,,,,)" + eol + // line 5 + R"(124,"a\"\"b\"",,,,,,)" + eol + // line 6 + R"(,,,1970-01-01,,,,"jklm")" + eol + // line 7 + R"(,,,,1970-01-02,,,)" + eol + // line 8 + R"(,,,,,2004-02-29 01:02:03,,)" + eol + // line 9 + R"(,,,,,,3600,)" + eol + // line 10 + R"(,"NA",,,,,,)" + eol; // line 11 +} + std::vector GenerateTestCases() { // Dummy schema and data for testing invalid options. auto dummy_schema = schema({field("a", uint8())}); @@ -306,6 +323,29 @@ std::vector GenerateTestCases() { /*delimiter=*/',', /*batch_size=*/5, /*quoting_header=*/QuotingStyle::None), "", expected_status_no_quotes_with_structural_in_header("b\"")}, + // EscapeStyle::Backslash: quotes escaped by a preceding backslash. + {abc_schema, populated_batch, + DefaultTestOptions(/*include_header=*/false, /*null_string=*/"", + QuotingStyle::Needed, /*eol=*/"\n", /*delimiter=*/',', + /*batch_size=*/5, /*quoting_header=*/QuotingStyle::Needed, + /*escape_style=*/EscapeStyle::Backslash), + UtilGetExpectedWithEOLBackslash("\n")}, + // EscapeStyle::Backslash with header (field name "b\"" escaped as "b\\\""). + {abc_schema, populated_batch, + DefaultTestOptions(/*include_header=*/true, /*null_string=*/"", + QuotingStyle::Needed, /*eol=*/"\n", /*delimiter=*/',', + /*batch_size=*/5, /*quoting_header=*/QuotingStyle::Needed, + /*escape_style=*/EscapeStyle::Backslash), + R"("a","b\"","c ","d","e","f","g","h")" + "\n" + + UtilGetExpectedWithEOLBackslash("\n")}, + // EscapeStyle::None: quotes are not escaped. + {schema({field("a", utf8())}), R"([{"a": "x\"y"}])", + DefaultTestOptions(/*include_header=*/false, /*null_string=*/"", + QuotingStyle::Needed, /*eol=*/"\n", /*delimiter=*/',', + /*batch_size=*/5, /*quoting_header=*/QuotingStyle::Needed, + /*escape_style=*/EscapeStyle::None), + R"("x"y")" "\n"}, }; }