latent proj bias

Author: stduhpf

latent-preview.h

@@ -1,4 +1,3 @@
-
 // https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/latent_formats.py#L152-L169
 const float flux_latent_rgb_proj[16][3] = {
     {-0.0346f, 0.0244f, 0.0681f},
@@ -17,6 +16,7 @@ const float flux_latent_rgb_proj[16][3] = {
     {-0.1264f, -0.0522f, -0.1103f},
     {-0.0280f, -0.0881f, -0.0499f},
     {-0.1262f, -0.0982f, -0.0778f}};
+float flux_latent_rgb_bias[3] = {-0.0329, -0.0718, -0.0851};
 
 // https://github.com/Stability-AI/sd3.5/blob/main/sd3_impls.py#L228-L246
 const float sd3_latent_rgb_proj[16][3] = {
@@ -37,22 +37,25 @@ const float sd3_latent_rgb_proj[16][3] = {
     {-0.0749f, -0.0634f, -0.0456f},
     {-0.1418f, -0.1457f, -0.1259f},
 };
+float sd3_latent_rgb_bias[3] = {0, 0, 0};
 
 // https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/latent_formats.py#L32-L38
 const float sdxl_latent_rgb_proj[4][3] = {
     {0.3651f, 0.4232f, 0.4341f},
     {-0.2533f, -0.0042f, 0.1068f},
     {0.1076f, 0.1111f, -0.0362f},
     {-0.3165f, -0.2492f, -0.2188f}};
+float sdxl_latent_rgb_bias[3] = {0.1084, -0.0175, -0.0011};
 
 // https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/latent_formats.py#L32-L38
 const float sd_latent_rgb_proj[4][3]{
     {0.3512f, 0.2297f, 0.3227f},
     {0.3250f, 0.4974f, 0.2350f},
     {-0.2829f, 0.1762f, 0.2721f},
     {-0.2120f, -0.2616f, -0.7177f}};
+float sd_latent_rgb_bias[3] = {0,0,0};
 
-void preview_latent_image(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], int width, int height, int dim) {
+void preview_latent_image(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int width, int height, int dim) {
     size_t buffer_head = 0;
     for (int j = 0; j < height; j++) {
         for (int i = 0; i < width; i++) {
@@ -64,6 +67,10 @@ void preview_latent_image(uint8_t* buffer, struct ggml_tensor* latents, const fl
                 g += value * latent_rgb_proj[d][1];
                 b += value * latent_rgb_proj[d][2];
             }
+                // bias
+                r += latent_rgb_bias[0];
+                g += latent_rgb_bias[1];
+                b += latent_rgb_bias[2];
 
             // change range
             r = r * .5f + .5f;

stable-diffusion.cpp

@@ -883,14 +883,17 @@ class StableDiffusionGGML {
         uint32_t dim           = latents->ne[2];
         if (preview_mode == PREVIEW_PROJ) {
             const float (*latent_rgb_proj)[channel];
+            float *latent_rgb_bias;
 
             if (dim == 16) {
                 // 16 channels VAE -> Flux or SD3
 
                 if (sd_version_is_sd3(version)) {
                     latent_rgb_proj = sd3_latent_rgb_proj;
+                    latent_rgb_bias = sd3_latent_rgb_bias;
                 } else if (sd_version_is_flux(version)) {
                     latent_rgb_proj = flux_latent_rgb_proj;
+                    latent_rgb_bias = flux_latent_rgb_bias;
                 } else {
                     LOG_WARN("No latent to RGB projection known for this model");
                     // unknown model
@@ -901,8 +904,10 @@ class StableDiffusionGGML {
                 // 4 channels VAE
                 if (sd_version_is_sdxl(version)) {
                     latent_rgb_proj = sdxl_latent_rgb_proj;
+                    latent_rgb_bias = sdxl_latent_rgb_bias;
                 } else if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
                     latent_rgb_proj = sd_latent_rgb_proj;
+                    latent_rgb_bias = sd_latent_rgb_bias;
                 } else {
                     // unknown model
                     LOG_WARN("No latent to RGB projection known for this model");
@@ -915,7 +920,7 @@ class StableDiffusionGGML {
             }
             uint8_t* data = (uint8_t*)malloc(width * height * channel * sizeof(uint8_t));
 
-            preview_latent_image(data, latents, latent_rgb_proj, width, height, dim);
+            preview_latent_image(data, latents, latent_rgb_proj,latent_rgb_bias, width, height, dim);
             sd_image_t image = {
                 width,
                 height,