BREAKING: Replace xml_builder and sweet_xml with Saxy

Be sure to update your code to use the new decoded format, keys on parsed XML are no longer atoms (for safety reasons)

The primary reason for this change is to prevent a possible memory leak from server crafted XML documents that would exhaust the atom limit, in the first place parsing XML to unsafe atoms was a BAD IDEA.

Author: IceDragon200

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Breaking Changes
 
+* Changed underlying XML encoding and decoder library to `saxy`, while encoding should remain the same, those handling XML responses must change their code to accept Saxy's "simple" format.
+  * If you use the `normalize_xml` option, keys are no longer atoms but strings.
 * Removed `:jsonapi` response body type, it will just report as `:json` instead
 * Remove `mode` option, do not use it the respective connection manager will set the mode for itself
 * ContentClient will now return `:unaccepted` and `{:malformed, term}`` in addition to :unk to differentiate response bodies.

lib/pepper/http/body_decoder/xml.ex

@@ -2,12 +2,12 @@ defmodule Pepper.HTTP.BodyDecoder.XML do
   import Pepper.HTTP.Utils
 
   def decode_body(body, options) do
-    data = SweetXml.parse(body)
     # Parse XML
+    {:ok, doc} = Saxy.SimpleForm.parse_string(body)
     if options[:normalize_xml] do
-      {:xmldoc, handle_xml_body(data)}
+      {:xmldoc, handle_xml_body(doc)}
     else
-      {:xml, data}
+      {:xml, doc}
     end
   end
 end

lib/pepper/http/body_encoder/xml.ex

@@ -1,6 +1,6 @@
 defmodule Pepper.HTTP.BodyEncoder.XML do
   def encode_body(term, _options) do
-    blob = XmlBuilder.generate(term, format: :none)
+    blob = Saxy.encode!(term)
     headers = [{"content-type", "application/xml"}]
     {:ok, {headers, blob}}
   end

lib/pepper/http/utils.ex

@@ -8,8 +8,6 @@ defmodule Pepper.HTTP.Utils do
 
   import Mint.HTTP1.Parse
 
-  require SweetXml
-
   @type http_method :: String.t()
                      | :connect
                      | :delete
@@ -120,59 +118,29 @@ defmodule Pepper.HTTP.Utils do
     :crypto.strong_rand_bytes(len)
   end
 
-  def handle_xml_body(doc) do
+  def handle_xml_body(doc) when is_tuple(doc) or is_list(doc) do
     doc =
       doc
       |> List.wrap()
-      |> Enum.map(fn item ->
-        record_type = elem(item, 0)
-        xml_item_to_map(record_type, item)
+      |> Enum.map(fn {_elem_name, _attributes, _children} = item ->
+        xml_item_to_map(item)
       end)
       |> deflate_xml_map()
 
     doc
   end
 
-  for name <- [
-      :xmlDecl,
-      :xmlAttribute,
-      :xmlNamespace,
-      :xmlNsNode,
-      :xmlElement,
-      :xmlText,
-      :xmlComment,
-      :xmlPI,
-      :xmlDocument,
-      :xmlObj,
-    ] do
-    def xml_item_to_map(unquote(name), item) do
-      SweetXml.unquote(name)(item)
-      |> xml_item_deep_to_map(unquote(name))
-    end
+  @spec xml_item_to_map(tuple()) :: tuple()
+  def xml_item_to_map({elem_name, _attributes, children}) do
+    {elem_name, Enum.map(children, fn item ->
+      xml_item_to_map(item)
+    end)}
   end
 
-  def xml_item_deep_to_map(item, :xmlElement) do
-    #namespace = xml_item_to_map(:xmlNamespace, item[:namespace])
-    #item = put_in(item[:namespace], namespace)
-    #put_in(item[:content], Enum.map(item[:content], fn item ->
-    #  xml_item_to_map(elem(item, 0), item)
-    #end))
-
-    {item[:expanded_name],
-      Enum.map(item[:content], fn item ->
-        xml_item_to_map(elem(item, 0), item)
-      end)
-    }
-  end
-
-  def xml_item_deep_to_map(item, :xmlNamespace) do
+  def xml_item_to_map(item) do
     item
   end
 
-  def xml_item_deep_to_map(item, :xmlText) do
-    to_string(item[:value])
-  end
-
   def deflate_xml_map([{_, _} | _] = list) when is_list(list) do
     [
       Enum.reduce(list, %{}, fn

mix.exs

@@ -35,18 +35,19 @@ defmodule Pepper.HTTP.MixProject do
       {:plug, "~> 1.6"},
       # JSON Parser
       {:jason, "~> 1.2"},
-      # XML Parser
-      {:sweet_xml, "~> 0.6"},
-      # XML Encoder
-      {:xml_builder, "~> 2.0"},
+      # XML Decoder / Encoder
+      {:saxy, "~> 1.5"},
       # CSV
       {:csv, "~> 2.0 or ~> 3.0"},
       # HTTP Library
-      {:mint, "~> 1.0"},
+      #{:mint, "~> 1.0"},
+      {:mint, path: "../mint"},
       {:httpoison, "~> 1.0"},
       # Certificate Store
       {:castore, "~> 0.1 or ~> 1.0"},
       {:bypass, "~> 1.0 or ~> 2.1", [only: :test]},
+      # benchmarking
+      {:benchee, "~> 1.1", only: [:dev]},
     ]
   end
 

mix.lock

@@ -1,12 +1,14 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"},
+  "benchee": {:hex, :benchee, "1.3.0", "f64e3b64ad3563fa9838146ddefb2d2f94cf5b473bdfd63f5ca4d0657bf96694", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "34f4294068c11b2bd2ebf2c59aac9c7da26ffa0068afdf3419f1b176e16c5f81"},
   "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
   "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
   "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
   "csv": {:hex, :csv, "3.2.1", "6d401f1ed33acb2627682a9ab6021e96d33ca6c1c6bccc243d8f7e2197d032f5", [:mix], [], "hexpm", "8f55a0524923ae49e97ff2642122a2ce7c61e159e7fe1184670b2ce847aee6c8"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
   "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
@@ -21,9 +23,9 @@
   "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"},
   "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
+  "saxy": {:hex, :saxy, "1.5.0", "0141127f2d042856f135fb2d94e0beecda7a2306f47546dbc6411fc5b07e28bf", [:mix], [], "hexpm", "ea7bb6328fbd1f2aceffa3ec6090bfb18c85aadf0f8e5030905e84235861cf89"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
-  "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},
+  "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
   "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
-  "xml_builder": {:hex, :xml_builder, "2.2.0", "cc5f1eeefcfcde6e90a9b77fb6c490a20bc1b856a7010ce6396f6da9719cbbab", [:mix], [], "hexpm", "9d66d52fb917565d358166a4314078d39ef04d552904de96f8e73f68f64a62c9"},
 }

test/pepper/http/content_client/accept_header_test.exs

@@ -296,8 +296,8 @@ defmodule Pepper.HTTP.ContentClient.AcceptHeaderTest do
       )
 
     assert [%{
-      :Response => [%{
-        :Status => ["OK"]
+      "Response" => [%{
+        "Status" => ["OK"]
       }]
     }] = doc
   end

test/pepper/http/content_client_test.exs

@@ -143,14 +143,15 @@ defmodule Pepper.HTTP.ContentClientTest do
 
     query_params = []
 
-    body =
-      {:xml,
-       XmlBuilder.document([
-         {:head, [],
-          [
-            {:ref, [], ["Test Value"]}
-          ]}
-       ])}
+    body = {:xml,
+      {
+        "head",
+        [],
+        [
+          {"ref", [], ["Test Value"]}
+        ]
+      }
+    }
 
     assert {:ok, %{protocol: ^protocol, status_code: 204}, {:unk, ""}} =
       Client.request(

test/support/plug_pipeline_helpers.ex

@@ -28,7 +28,7 @@ defmodule Pepper.HTTP.Support.PlugPipelineHelpers do
     def parse_xml(conn, _) do
       {:ok, body, conn} = Plug.Conn.read_body(conn)
       try do
-        doc = SweetXml.parse(body)
+        doc = Saxy.SimpleForm.parse_string(body)
         put_in(conn.params["_xml"], doc)
       rescue ex ->
         raise Plug.Parsers.ParseError, exception: ex