feat: Add encoding parameter to fetch function

- Adds a new parameter `response_encoding` to the fetch function. - This
parameter allows the user to specify a charset for decoding the response
body. - It supports all standard web encodings, and also `hex` and
`base64`. - If no encoding is specified, the response is decoded as
utf-8, or as base64 if it contains binary data.

Author: lovasoa

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG.md
 
+## v0.37.0
+ - Added a new parameter `encoding` to the [fetch](https://sql-page.com/functions.sql?function=fetch) function:
+  - All [standard web encodings](https://encoding.spec.whatwg.org/#concept-encoding-get) are supported.
+  - Additionally, `base64` can be specified to decode binary data as base64 (compatible with [data URI](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs))
+  - By default, the old behavior of the `fetch_with_meta` function is preserved: the response body is decoded as `utf-8` if possible, otherwise the response is encoded in `base64`.
+
 ## v0.36.1
  - Fix regression introduced in v0.36.0: PostgreSQL money values showed as 0.0
    - The recommended way to display money values in postgres is still to format them in the way you expect in SQL. See https://github.com/sqlpage/SQLPage/issues/983

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "sqlpage"
-version = "0.36.1"
+version = "0.37.0"
 edition = "2021"
 description = "Build data user interfaces entirely in SQL. A web server that takes .sql files and formats the query result using pre-made configurable professional-looking components."
 keywords = ["web", "sql", "framework"]
@@ -73,6 +73,7 @@ awc = { version = "3", features = ["rustls-0_23-webpki-roots"] }
 clap = { version = "4.5.17", features = ["derive"] }
 tokio-util = "0.7.12"
 openidconnect = { version = "4.0.0", default-features = false }
+encoding_rs = "0.8.35"
 
 [build-dependencies]
 awc = { version = "3", features = ["rustls-0_23-webpki-roots"] }

Cargo.toml

@@ -87,6 +87,7 @@ The fetch function accepts either a URL string, or a JSON object with the follow
  - `timeout_ms`: The maximum time to wait for the request, in milliseconds. Defaults to 5000.
  - `username`: Optional username for HTTP Basic Authentication. Introduced in version 0.33.0.
  - `password`: Optional password for HTTP Basic Authentication. Only used if username is provided. Introduced in version 0.33.0.
+ - `response_encoding`: Optional charset to use for decoding the response body. Defaults to `utf8`, or `base64` if the response contains binary data. All [standard web encodings](https://encoding.spec.whatwg.org/#concept-encoding-get) are supported, plus `hex`, `base64`, and `base64url`. Introduced in version 0.37.0.
 
 # Error handling and reading response headers
 

examples/official-site/sqlpage/migrations/40_fetch.sql

@@ -11,6 +11,7 @@ use crate::webserver::{
 use anyhow::{anyhow, Context};
 use futures_util::StreamExt;
 use mime_guess::mime;
+use std::fmt::Write;
 use std::{borrow::Cow, ffi::OsStr, str::FromStr};
 
 super::function_definition_macro::sqlpage_functions! {
@@ -213,13 +214,46 @@ async fn fetch(
             )
         })?
         .to_vec();
-    let response_str = String::from_utf8(body).with_context(
-        || format!("Unable to convert the response from {} to a string. Only UTF-8 responses are supported.", http_request.url),
-    )?;
+    let response_str = decode_response(body, http_request.response_encoding.as_deref())?;
     log::debug!("Fetch response: {response_str}");
     Ok(response_str)
 }
 
+fn decode_response(response: Vec<u8>, encoding: Option<&str>) -> anyhow::Result<String> {
+    match encoding {
+        Some("base64") => Ok(base64::Engine::encode(
+            &base64::engine::general_purpose::STANDARD,
+            response,
+        )),
+        Some("base64url") => Ok(base64::Engine::encode(
+            &base64::engine::general_purpose::URL_SAFE,
+            response,
+        )),
+        Some("hex") => Ok(response.into_iter().fold(String::new(), |mut acc, byte| {
+            write!(&mut acc, "{byte:02x}").unwrap();
+            acc
+        })),
+        Some(encoding_label) => Ok(encoding_rs::Encoding::for_label(encoding_label.as_bytes())
+            .with_context(|| format!("Invalid encoding name: {encoding_label}"))?
+            .decode(&response)
+            .0
+            .into_owned()),
+        None => {
+            let body_str = String::from_utf8(response);
+            match body_str {
+                Ok(body_str) => Ok(body_str),
+                Err(decoding_error) => {
+                    log::warn!("fetch(...) response is not UTF-8 and no encoding was specified. Decoding the response as base64. Please explicitly set the encoding to \"base64\" if this is the expected behavior.");
+                    Ok(base64::Engine::encode(
+                        &base64::engine::general_purpose::STANDARD,
+                        decoding_error.into_bytes(),
+                    ))
+                }
+            }
+        }
+    }
+}
+
 async fn fetch_with_meta(
     request: &RequestInfo,
     http_request: super::http_fetch_request::HttpFetchRequest<'_>,
@@ -270,27 +304,15 @@ async fn fetch_with_meta(
             match response.body().await {
                 Ok(body) => {
                     let body_bytes = body.to_vec();
-                    let body_str = String::from_utf8(body_bytes);
-
-                    match body_str {
-                        Ok(body_str) if is_json => {
-                            obj.serialize_entry(
-                                "body",
-                                &serde_json::value::RawValue::from_string(body_str)?,
-                            )?;
-                        }
-                        Ok(body_str) => {
-                            obj.serialize_entry("body", &body_str)?;
-                        }
-                        Err(utf8_err) => {
-                            let mut base64_string = String::new();
-                            base64::Engine::encode_string(
-                                &base64::engine::general_purpose::STANDARD,
-                                utf8_err.as_bytes(),
-                                &mut base64_string,
-                            );
-                            obj.serialize_entry("body", &base64_string)?;
-                        }
+                    let body_str =
+                        decode_response(body_bytes, http_request.response_encoding.as_deref())?;
+                    if is_json {
+                        obj.serialize_entry(
+                            "json_body",
+                            &serde_json::value::RawValue::from_string(body_str)?,
+                        )?;
+                    } else {
+                        obj.serialize_entry("body", &body_str)?;
                     }
                 }
                 Err(e) => {

src/webserver/database/sqlpage_functions/functions.rs

@@ -38,6 +38,7 @@ pub(super) struct HttpFetchRequest<'b> {
     #[serde(borrow)]
     pub body: Option<Cow<'b, serde_json::value::RawValue>>,
     pub timeout_ms: Option<u64>,
+    pub response_encoding: Option<Cow<'b, str>>,
 }
 
 fn deserialize_map_to_vec_pairs<'de, D: serde::Deserializer<'de>>(
@@ -78,6 +79,7 @@ impl<'a> BorrowFromStr<'a> for HttpFetchRequest<'a> {
                 password: None,
                 body: None,
                 timeout_ms: None,
+                response_encoding: None,
             }
         } else {
             match s {
@@ -104,6 +106,7 @@ impl HttpFetchRequest<'_> {
             timeout_ms: self.timeout_ms,
             username: self.username.map(Cow::into_owned).map(Cow::Owned),
             password: self.password.map(Cow::into_owned).map(Cow::Owned),
+            response_encoding: self.response_encoding.map(Cow::into_owned).map(Cow::Owned),
         }
     }
 }

src/webserver/database/sqlpage_functions/http_fetch_request.rs

@@ -0,0 +1,9 @@
+set res = sqlpage.fetch('{
+  "url": "http://localhost:' || $echo_port || '/hello_world",
+  "response_encoding": "base64"
+}');
+select 'text' as component,
+    case
+        when $res LIKE 'R0VUIC9oZWxsb193b3Js%' then 'It works !'
+        else 'It failed ! Got: ' || $res
+    end as contents;

tests/sql_test_files/it_works_fetch_base64.sql

@@ -0,0 +1,9 @@
+set res = sqlpage.fetch('{
+  "url": "http://localhost:' || $echo_port || '/hello_world",
+  "response_encoding": "hex"
+}');
+select 'text' as component,
+    case
+        when $res LIKE '474554202f68656c6c6f5f776f726c64%' then 'It works !'
+        else 'It failed ! Got: ' || $res
+    end as contents;