From c064dcf77222732b10798b9b05d0f79ad9bf950a Mon Sep 17 00:00:00 2001 From: ThanhDungCR7 Date: Thu, 4 Apr 2024 13:38:44 +0700 Subject: [PATCH 1/2] update --- generation_models/configs/model_config.yaml | 5 + generation_models/face_to_many.json | 437 +++++++++++++++++ generation_models/niche_go_journey.py | 1 + generation_models/niche_stable_diffusion.py | 20 + .../niche_stable_diffusion_xl.py | 2 + generation_models/niche_sticker_maker.py | 36 ++ generation_models/sticker_maker.json | 455 ++++++++++++++++++ generation_models/utils_api_comfyui.py | 179 +++++++ test.py | 21 + 9 files changed, 1156 insertions(+) create mode 100644 generation_models/face_to_many.json create mode 100644 generation_models/niche_sticker_maker.py create mode 100644 generation_models/sticker_maker.json create mode 100644 generation_models/utils_api_comfyui.py create mode 100644 test.py diff --git a/generation_models/configs/model_config.yaml b/generation_models/configs/model_config.yaml index 98ab7919..7107f2c5 100644 --- a/generation_models/configs/model_config.yaml +++ b/generation_models/configs/model_config.yaml @@ -37,6 +37,11 @@ GoJourney: params: supporting_pipelines: - "gojourney" +StickerMaker: + target: generation_models.NicheStickerMaker + params: + supporting_pipelines: + - "txt2img" Gemma7b: target: "" repo_id: "google/gemma-7b-it" diff --git a/generation_models/face_to_many.json b/generation_models/face_to_many.json new file mode 100644 index 00000000..dc0762cd --- /dev/null +++ b/generation_models/face_to_many.json @@ -0,0 +1,437 @@ +{ + "2": { + "inputs": { + "ckpt_name": "albedobaseXL_v13.safetensors", + "vae_name": "Baked VAE", + "clip_skip": -1, + "lora_name": "None", + "lora_model_strength": 1, + "lora_clip_strength": 1, + "positive": "3D Render Style, 3DRenderAF, person", + "negative": "photo, photography, nsfw, nude, ugly, broken, watermark", + "token_normalization": "none", + "weight_interpretation": "comfy", + "empty_latent_width": 1024, + "empty_latent_height": 1024, + "batch_size": 1, + "lora_stack": [ + "3", + 0 + ], + "cnet_stack": [ + "28", + 0 + ] + }, + "class_type": "Efficient Loader", + "_meta": { + "title": "Efficient Loader" + } + }, + "3": { + "inputs": { + "input_mode": "simple", + "lora_count": 3, + "lora_name_1": "artificialguybr/3DRedmond-3DRenderStyle-3DRenderAF.safetensors", + "lora_wt_1": 1, + "model_str_1": 1, + "clip_str_1": 1, + "lora_name_2": "None", + "lora_wt_2": 1, + "model_str_2": 1, + "clip_str_2": 1, + "lora_name_3": "None", + "lora_wt_3": 1, + "model_str_3": 1, + "clip_str_3": 1, + "lora_name_4": "None", + "lora_wt_4": 1, + "model_str_4": 1, + "clip_str_4": 1, + "lora_name_5": "None", + "lora_wt_5": 1, + "model_str_5": 1, + "clip_str_5": 1, + "lora_name_6": "None", + "lora_wt_6": 1, + "model_str_6": 1, + "clip_str_6": 1, + "lora_name_7": "None", + "lora_wt_7": 1, + "model_str_7": 1, + "clip_str_7": 1, + "lora_name_8": "None", + "lora_wt_8": 1, + "model_str_8": 1, + "clip_str_8": 1, + "lora_name_9": "None", + "lora_wt_9": 1, + "model_str_9": 1, + "clip_str_9": 1, + "lora_name_10": "None", + "lora_wt_10": 1, + "model_str_10": 1, + "clip_str_10": 1, + "lora_name_11": "None", + "lora_wt_11": 1, + "model_str_11": 1, + "clip_str_11": 1, + "lora_name_12": "None", + "lora_wt_12": 1, + "model_str_12": 1, + "clip_str_12": 1, + "lora_name_13": "None", + "lora_wt_13": 1, + "model_str_13": 1, + "clip_str_13": 1, + "lora_name_14": "None", + "lora_wt_14": 1, + "model_str_14": 1, + "clip_str_14": 1, + "lora_name_15": "None", + "lora_wt_15": 1, + "model_str_15": 1, + "clip_str_15": 1, + "lora_name_16": "None", + "lora_wt_16": 1, + "model_str_16": 1, + "clip_str_16": 1, + "lora_name_17": "None", + "lora_wt_17": 1, + "model_str_17": 1, + "clip_str_17": 1, + "lora_name_18": "None", + "lora_wt_18": 1, + "model_str_18": 1, + "clip_str_18": 1, + "lora_name_19": "None", + "lora_wt_19": 1, + "model_str_19": 1, + "clip_str_19": 1, + "lora_name_20": "None", + "lora_wt_20": 1, + "model_str_20": 1, + "clip_str_20": 1, + "lora_name_21": "None", + "lora_wt_21": 1, + "model_str_21": 1, + "clip_str_21": 1, + "lora_name_22": "None", + "lora_wt_22": 1, + "model_str_22": 1, + "clip_str_22": 1, + "lora_name_23": "None", + "lora_wt_23": 1, + "model_str_23": 1, + "clip_str_23": 1, + "lora_name_24": "None", + "lora_wt_24": 1, + "model_str_24": 1, + "clip_str_24": 1, + "lora_name_25": "None", + "lora_wt_25": 1, + "model_str_25": 1, + "clip_str_25": 1, + "lora_name_26": "None", + "lora_wt_26": 1, + "model_str_26": 1, + "clip_str_26": 1, + "lora_name_27": "None", + "lora_wt_27": 1, + "model_str_27": 1, + "clip_str_27": 1, + "lora_name_28": "None", + "lora_wt_28": 1, + "model_str_28": 1, + "clip_str_28": 1, + "lora_name_29": "None", + "lora_wt_29": 1, + "model_str_29": 1, + "clip_str_29": 1, + "lora_name_30": "None", + "lora_wt_30": 1, + "model_str_30": 1, + "clip_str_30": 1, + "lora_name_31": "None", + "lora_wt_31": 1, + "model_str_31": 1, + "clip_str_31": 1, + "lora_name_32": "None", + "lora_wt_32": 1, + "model_str_32": 1, + "clip_str_32": 1, + "lora_name_33": "None", + "lora_wt_33": 1, + "model_str_33": 1, + "clip_str_33": 1, + "lora_name_34": "None", + "lora_wt_34": 1, + "model_str_34": 1, + "clip_str_34": 1, + "lora_name_35": "None", + "lora_wt_35": 1, + "model_str_35": 1, + "clip_str_35": 1, + "lora_name_36": "None", + "lora_wt_36": 1, + "model_str_36": 1, + "clip_str_36": 1, + "lora_name_37": "None", + "lora_wt_37": 1, + "model_str_37": 1, + "clip_str_37": 1, + "lora_name_38": "None", + "lora_wt_38": 1, + "model_str_38": 1, + "clip_str_38": 1, + "lora_name_39": "None", + "lora_wt_39": 1, + "model_str_39": 1, + "clip_str_39": 1, + "lora_name_40": "None", + "lora_wt_40": 1, + "model_str_40": 1, + "clip_str_40": 1, + "lora_name_41": "None", + "lora_wt_41": 1, + "model_str_41": 1, + "clip_str_41": 1, + "lora_name_42": "None", + "lora_wt_42": 1, + "model_str_42": 1, + "clip_str_42": 1, + "lora_name_43": "None", + "lora_wt_43": 1, + "model_str_43": 1, + "clip_str_43": 1, + "lora_name_44": "None", + "lora_wt_44": 1, + "model_str_44": 1, + "clip_str_44": 1, + "lora_name_45": "None", + "lora_wt_45": 1, + "model_str_45": 1, + "clip_str_45": 1, + "lora_name_46": "None", + "lora_wt_46": 1, + "model_str_46": 1, + "clip_str_46": 1, + "lora_name_47": "None", + "lora_wt_47": 1, + "model_str_47": 1, + "clip_str_47": 1, + "lora_name_48": "None", + "lora_wt_48": 1, + "model_str_48": 1, + "clip_str_48": 1, + "lora_name_49": "None", + "lora_wt_49": 1, + "model_str_49": 1, + "clip_str_49": 1 + }, + "class_type": "LoRA Stacker", + "_meta": { + "title": "LoRA Stacker" + } + }, + "4": { + "inputs": { + "seed": 1051763972092850, + "steps": 17, + "cfg": 7, + "sampler_name": "dpmpp_2m_sde_gpu", + "scheduler": "karras", + "denoise": 0.65, + "preview_method": "none", + "vae_decode": "true", + "model": [ + "41", + 0 + ], + "positive": [ + "41", + 1 + ], + "negative": [ + "41", + 2 + ], + "latent_image": [ + "51", + 0 + ], + "optional_vae": [ + "2", + 4 + ] + }, + "class_type": "KSampler (Efficient)", + "_meta": { + "title": "KSampler (Efficient)" + } + }, + "5": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "4", + 5 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "22": { + "inputs": { + "image": "Screenshot 2024-02-26 at 16.13.24.png", + "upload": "image" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "24": { + "inputs": { + "control_net_name": "depth-zoe-xl-v1.0-controlnet.safetensors" + }, + "class_type": "ControlNetLoader", + "_meta": { + "title": "Load ControlNet Model" + } + }, + "28": { + "inputs": { + "strength": 0.8, + "start_percent": 0, + "end_percent": 1, + "control_net": [ + "24", + 0 + ], + "image": [ + "49", + 0 + ] + }, + "class_type": "Control Net Stacker", + "_meta": { + "title": "Control Net Stacker" + } + }, + "35": { + "inputs": { + "instantid_file": "instantid-ip-adapter.bin" + }, + "class_type": "InstantIDModelLoader", + "_meta": { + "title": "Load InstantID Model" + } + }, + "36": { + "inputs": { + "provider": "CUDA" + }, + "class_type": "InstantIDFaceAnalysis", + "_meta": { + "title": "InstantID Face Analysis" + } + }, + "40": { + "inputs": { + "control_net_name": "instantid-controlnet.safetensors" + }, + "class_type": "ControlNetLoader", + "_meta": { + "title": "Load ControlNet Model" + } + }, + "41": { + "inputs": { + "weight": 1, + "start_at": 0, + "end_at": 1, + "instantid": [ + "35", + 0 + ], + "insightface": [ + "36", + 0 + ], + "control_net": [ + "40", + 0 + ], + "image": [ + "67", + 0 + ], + "model": [ + "2", + 0 + ], + "positive": [ + "2", + 1 + ], + "negative": [ + "2", + 2 + ] + }, + "class_type": "ApplyInstantID", + "_meta": { + "title": "Apply InstantID" + } + }, + "49": { + "inputs": { + "preprocessor": "MiDaS-DepthMapPreprocessor", + "resolution": 512, + "image": [ + "67", + 0 + ] + }, + "class_type": "AIO_Preprocessor", + "_meta": { + "title": "AIO Aux Preprocessor" + } + }, + "51": { + "inputs": { + "pixels": [ + "67", + 0 + ], + "vae": [ + "2", + 4 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode" + } + }, + "67": { + "inputs": { + "width": 1024, + "height": 1024, + "interpolation": "nearest", + "keep_proportion": true, + "condition": "always", + "image": [ + "22", + 0 + ] + }, + "class_type": "ImageResize+", + "_meta": { + "title": "🔧 Image Resize" + } + } +} \ No newline at end of file diff --git a/generation_models/niche_go_journey.py b/generation_models/niche_go_journey.py index 5cd3fc5d..7a816094 100644 --- a/generation_models/niche_go_journey.py +++ b/generation_models/niche_go_journey.py @@ -18,6 +18,7 @@ def load_model(self, *args, **kwargs): def __call__(self, *args, **kwargs): return self.inference_function(*args, **kwargs) + def load_imagine(self, *args, **kwargs): imagine_endpoint = "https://api.midjourneyapi.xyz/mj/v2/imagine" fetch_endpoint = "https://api.midjourneyapi.xyz/mj/v2/fetch" diff --git a/generation_models/niche_stable_diffusion.py b/generation_models/niche_stable_diffusion.py index b1b2a521..86703b46 100644 --- a/generation_models/niche_stable_diffusion.py +++ b/generation_models/niche_stable_diffusion.py @@ -120,3 +120,23 @@ def process_conditional_image(self, **kwargs) -> Image.Image: resolution = kwargs.get("resolution", 512) conditional_image = resize_for_condition_image(conditional_image, resolution) return conditional_image + +if __name__ == "__main__": + params = { + "checkpoint_file": "checkpoints/RealisticVision.safetensors", + "download_url": "https://civitai.com/api/download/models/130072?type=Model&format=SafeTensor&size=pruned&fp=fp16", + "scheduler": "dpm++2m", + "supporting_pipelines": ['txt2img'] + } + pipe = NicheStableDiffusion( + **params + ) + + input_dict = { + "pipeline_type": "txt2img", + "prompt": "a cat", + "num_inference_steps": 25, + } + + image = pipe(**input_dict) + image.save("debug.webp") \ No newline at end of file diff --git a/generation_models/niche_stable_diffusion_xl.py b/generation_models/niche_stable_diffusion_xl.py index 02152806..f9a42c88 100644 --- a/generation_models/niche_stable_diffusion_xl.py +++ b/generation_models/niche_stable_diffusion_xl.py @@ -71,3 +71,5 @@ def process_conditional_image(self, **kwargs) -> Image.Image: resolution = kwargs.get("resolution", 768) conditional_image = resize_for_condition_image(conditional_image, resolution) return conditional_image + + diff --git a/generation_models/niche_sticker_maker.py b/generation_models/niche_sticker_maker.py new file mode 100644 index 00000000..7986c36d --- /dev/null +++ b/generation_models/niche_sticker_maker.py @@ -0,0 +1,36 @@ +from base_model import BaseModel +from generation_models.utils_api_comfyui import * + +class NicheStickerMaker(BaseModel): + def __init__(self, *args, **kwargs): + self.server_address = "82.67.70.191:40830" + self.client_id = str(uuid.uuid4()) + self.inference_function = self.load_model(*args, **kwargs) + + + def load_model(self, *args, **kwargs): + imagine_inference_function = self.load_image(*args, **kwargs) + return imagine_inference_function + + def load_image(self, *args, **kwargs): + workflow = load_workflow("sticker_maker.json") + + def inference_function(*args, **kwargs): + workflow["2"]["inputs"]["positive"] = kwargs["prompt"] + workflow["4"]["inputs"]["seed"] = seed + + + ws = websocket.WebSocket() + ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id)) + + + + images = get_images(ws, prompt) + return images + + return inference_function + +if __name__=="__main__": + test_sticker = NicheStickerMaker({"prompt":"cute dog", "seed":7}) + test_sticker.load_model() + diff --git a/generation_models/sticker_maker.json b/generation_models/sticker_maker.json new file mode 100644 index 00000000..c116120f --- /dev/null +++ b/generation_models/sticker_maker.json @@ -0,0 +1,455 @@ +{ + "2": { + "inputs": { + "ckpt_name": "albedobaseXL_v13.safetensors", + "vae_name": "Baked VAE", + "clip_skip": -1, + "lora_name": "None", + "lora_model_strength": 1, + "lora_clip_strength": 1, + "positive": "Sticker, cute cat, svg, solid color background", + "negative": "nsfw, nude, ugly, broken, photo, photography", + "token_normalization": "none", + "weight_interpretation": "comfy", + "empty_latent_width": 1024, + "empty_latent_height": 1024, + "batch_size": 1, + "lora_stack": [ + "3", + 0 + ] + }, + "class_type": "Efficient Loader", + "_meta": { + "title": "Efficient Loader" + } + }, + "3": { + "inputs": { + "input_mode": "simple", + "lora_count": 3, + "lora_name_1": "artificialguybr/StickersRedmond.safetensors", + "lora_wt_1": 1, + "model_str_1": 1, + "clip_str_1": 1, + "lora_name_2": "None", + "lora_wt_2": 1, + "model_str_2": 1, + "clip_str_2": 1, + "lora_name_3": "None", + "lora_wt_3": 1, + "model_str_3": 1, + "clip_str_3": 1, + "lora_name_4": "None", + "lora_wt_4": 1, + "model_str_4": 1, + "clip_str_4": 1, + "lora_name_5": "None", + "lora_wt_5": 1, + "model_str_5": 1, + "clip_str_5": 1, + "lora_name_6": "None", + "lora_wt_6": 1, + "model_str_6": 1, + "clip_str_6": 1, + "lora_name_7": "None", + "lora_wt_7": 1, + "model_str_7": 1, + "clip_str_7": 1, + "lora_name_8": "None", + "lora_wt_8": 1, + "model_str_8": 1, + "clip_str_8": 1, + "lora_name_9": "None", + "lora_wt_9": 1, + "model_str_9": 1, + "clip_str_9": 1, + "lora_name_10": "None", + "lora_wt_10": 1, + "model_str_10": 1, + "clip_str_10": 1, + "lora_name_11": "None", + "lora_wt_11": 1, + "model_str_11": 1, + "clip_str_11": 1, + "lora_name_12": "None", + "lora_wt_12": 1, + "model_str_12": 1, + "clip_str_12": 1, + "lora_name_13": "None", + "lora_wt_13": 1, + "model_str_13": 1, + "clip_str_13": 1, + "lora_name_14": "None", + "lora_wt_14": 1, + "model_str_14": 1, + "clip_str_14": 1, + "lora_name_15": "None", + "lora_wt_15": 1, + "model_str_15": 1, + "clip_str_15": 1, + "lora_name_16": "None", + "lora_wt_16": 1, + "model_str_16": 1, + "clip_str_16": 1, + "lora_name_17": "None", + "lora_wt_17": 1, + "model_str_17": 1, + "clip_str_17": 1, + "lora_name_18": "None", + "lora_wt_18": 1, + "model_str_18": 1, + "clip_str_18": 1, + "lora_name_19": "None", + "lora_wt_19": 1, + "model_str_19": 1, + "clip_str_19": 1, + "lora_name_20": "None", + "lora_wt_20": 1, + "model_str_20": 1, + "clip_str_20": 1, + "lora_name_21": "None", + "lora_wt_21": 1, + "model_str_21": 1, + "clip_str_21": 1, + "lora_name_22": "None", + "lora_wt_22": 1, + "model_str_22": 1, + "clip_str_22": 1, + "lora_name_23": "None", + "lora_wt_23": 1, + "model_str_23": 1, + "clip_str_23": 1, + "lora_name_24": "None", + "lora_wt_24": 1, + "model_str_24": 1, + "clip_str_24": 1, + "lora_name_25": "None", + "lora_wt_25": 1, + "model_str_25": 1, + "clip_str_25": 1, + "lora_name_26": "None", + "lora_wt_26": 1, + "model_str_26": 1, + "clip_str_26": 1, + "lora_name_27": "None", + "lora_wt_27": 1, + "model_str_27": 1, + "clip_str_27": 1, + "lora_name_28": "None", + "lora_wt_28": 1, + "model_str_28": 1, + "clip_str_28": 1, + "lora_name_29": "None", + "lora_wt_29": 1, + "model_str_29": 1, + "clip_str_29": 1, + "lora_name_30": "None", + "lora_wt_30": 1, + "model_str_30": 1, + "clip_str_30": 1, + "lora_name_31": "None", + "lora_wt_31": 1, + "model_str_31": 1, + "clip_str_31": 1, + "lora_name_32": "None", + "lora_wt_32": 1, + "model_str_32": 1, + "clip_str_32": 1, + "lora_name_33": "None", + "lora_wt_33": 1, + "model_str_33": 1, + "clip_str_33": 1, + "lora_name_34": "None", + "lora_wt_34": 1, + "model_str_34": 1, + "clip_str_34": 1, + "lora_name_35": "None", + "lora_wt_35": 1, + "model_str_35": 1, + "clip_str_35": 1, + "lora_name_36": "None", + "lora_wt_36": 1, + "model_str_36": 1, + "clip_str_36": 1, + "lora_name_37": "None", + "lora_wt_37": 1, + "model_str_37": 1, + "clip_str_37": 1, + "lora_name_38": "None", + "lora_wt_38": 1, + "model_str_38": 1, + "clip_str_38": 1, + "lora_name_39": "None", + "lora_wt_39": 1, + "model_str_39": 1, + "clip_str_39": 1, + "lora_name_40": "None", + "lora_wt_40": 1, + "model_str_40": 1, + "clip_str_40": 1, + "lora_name_41": "None", + "lora_wt_41": 1, + "model_str_41": 1, + "clip_str_41": 1, + "lora_name_42": "None", + "lora_wt_42": 1, + "model_str_42": 1, + "clip_str_42": 1, + "lora_name_43": "None", + "lora_wt_43": 1, + "model_str_43": 1, + "clip_str_43": 1, + "lora_name_44": "None", + "lora_wt_44": 1, + "model_str_44": 1, + "clip_str_44": 1, + "lora_name_45": "None", + "lora_wt_45": 1, + "model_str_45": 1, + "clip_str_45": 1, + "lora_name_46": "None", + "lora_wt_46": 1, + "model_str_46": 1, + "clip_str_46": 1, + "lora_name_47": "None", + "lora_wt_47": 1, + "model_str_47": 1, + "clip_str_47": 1, + "lora_name_48": "None", + "lora_wt_48": 1, + "model_str_48": 1, + "clip_str_48": 1, + "lora_name_49": "None", + "lora_wt_49": 1, + "model_str_49": 1, + "clip_str_49": 1 + }, + "class_type": "LoRA Stacker", + "_meta": { + "title": "LoRA Stacker" + } + }, + "4": { + "inputs": { + "seed": 940769864734814, + "steps": 20, + "cfg": 7, + "sampler_name": "dpmpp_2m_sde", + "scheduler": "karras", + "denoise": 1, + "preview_method": "auto", + "vae_decode": "true", + "model": [ + "2", + 0 + ], + "positive": [ + "2", + 1 + ], + "negative": [ + "2", + 2 + ], + "latent_image": [ + "2", + 3 + ], + "optional_vae": [ + "2", + 4 + ] + }, + "class_type": "KSampler (Efficient)", + "_meta": { + "title": "KSampler (Efficient)" + } + }, + "5": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "4", + 5 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "8": { + "inputs": {}, + "class_type": "BRIA_RMBG_ModelLoader_Zho", + "_meta": { + "title": "🧹BRIA_RMBG Model Loader" + } + }, + "9": { + "inputs": { + "rmbgmodel": [ + "8", + 0 + ], + "image": [ + "4", + 5 + ] + }, + "class_type": "BRIA_RMBG_Zho", + "_meta": { + "title": "🧹BRIA RMBG" + } + }, + "10": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "9", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "11": { + "inputs": { + "upscale_by": 2, + "seed": 456216621533243, + "steps": 10, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 0.2, + "mode_type": "Linear", + "tile_width": 512, + "tile_height": 512, + "mask_blur": 8, + "tile_padding": 32, + "seam_fix_mode": "None", + "seam_fix_denoise": 1, + "seam_fix_width": 64, + "seam_fix_mask_blur": 8, + "seam_fix_padding": 16, + "force_uniform_tiles": true, + "tiled_decode": false, + "image": [ + "4", + 5 + ], + "model": [ + "12", + 0 + ], + "positive": [ + "13", + 0 + ], + "negative": [ + "14", + 0 + ], + "vae": [ + "12", + 2 + ], + "upscale_model": [ + "15", + 0 + ] + }, + "class_type": "UltimateSDUpscale", + "_meta": { + "title": "Ultimate SD Upscale" + } + }, + "12": { + "inputs": { + "ckpt_name": "dreamshaper_8.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "13": { + "inputs": { + "text": "svg sticker", + "clip": [ + "12", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "14": { + "inputs": { + "text": "nsfw", + "clip": [ + "12", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "15": { + "inputs": { + "model_name": "4x-AnimeSharp.pth" + }, + "class_type": "UpscaleModelLoader", + "_meta": { + "title": "Load Upscale Model" + } + }, + "16": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "11", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "17": { + "inputs": { + "rmbgmodel": [ + "8", + 0 + ], + "image": [ + "11", + 0 + ] + }, + "class_type": "BRIA_RMBG_Zho", + "_meta": { + "title": "🧹BRIA RMBG" + } + }, + "18": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "17", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +} \ No newline at end of file diff --git a/generation_models/utils_api_comfyui.py b/generation_models/utils_api_comfyui.py new file mode 100644 index 00000000..334df85c --- /dev/null +++ b/generation_models/utils_api_comfyui.py @@ -0,0 +1,179 @@ +import websocket #NOTE: websocket-client (https://github.com/websocket-client/websocket-client) +import uuid +import json +import urllib.request +import urllib.parse + +server_address = "82.67.70.191:40830" +client_id = str(uuid.uuid4()) + +def queue_prompt(prompt): + p = {"prompt": prompt, "client_id": client_id} + data = json.dumps(p).encode('utf-8') + req = urllib.request.Request("http://{}/prompt".format(server_address), data=data) + return json.loads(urllib.request.urlopen(req).read()) + +def get_image(filename, subfolder, folder_type): + data = {"filename": filename, "subfolder": subfolder, "type": folder_type} + url_values = urllib.parse.urlencode(data) + with urllib.request.urlopen("http://{}/view?{}".format(server_address, url_values)) as response: + return response.read() + +def get_history(prompt_id): + with urllib.request.urlopen("http://{}/history/{}".format(server_address, prompt_id)) as response: + return json.loads(response.read()) + +def load_workflow(workflow_path): + try: + with open(workflow_path, 'r') as file: + workflow = json.load(file) + return json.dumps(workflow) + except FileNotFoundError: + print(f"The file {workflow_path} was not found.") + return None + except json.JSONDecodeError: + print(f"The file {workflow_path} contains invalid JSON.") + return None + +def get_images(ws, prompt): + prompt_id = queue_prompt(prompt)['prompt_id'] + output_images = {} + while True: + out = ws.recv() + if isinstance(out, str): + message = json.loads(out) + if message['type'] == 'executing': + data = message['data'] + if data['node'] is None and data['prompt_id'] == prompt_id: + break #Execution is done + else: + continue #previews are binary data + + history = get_history(prompt_id)[prompt_id] + for o in history['outputs']: + for node_id in history['outputs']: + node_output = history['outputs'][node_id] + if 'images' in node_output: + images_output = [] + for image in node_output['images']: + image_data = get_image(image['filename'], image['subfolder'], image['type']) + images_output.append(image_data) + output_images[node_id] = images_output + + return output_images +if __name__ == "__main__": + prompt_text = """ + { + "3": { + "class_type": "KSampler", + "inputs": { + "cfg": 8, + "denoise": 1, + "latent_image": [ + "5", + 0 + ], + "model": [ + "4", + 0 + ], + "negative": [ + "7", + 0 + ], + "positive": [ + "6", + 0 + ], + "sampler_name": "euler", + "scheduler": "normal", + "seed": 8566257, + "steps": 20 + } + }, + "4": { + "class_type": "CheckpointLoaderSimple", + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly.ckpt" + } + }, + "5": { + "class_type": "EmptyLatentImage", + "inputs": { + "batch_size": 1, + "height": 512, + "width": 512 + } + }, + "6": { + "class_type": "CLIPTextEncode", + "inputs": { + "clip": [ + "4", + 1 + ], + "text": "masterpiece best quality girl" + } + }, + "7": { + "class_type": "CLIPTextEncode", + "inputs": { + "clip": [ + "4", + 1 + ], + "text": "bad hands" + } + }, + "8": { + "class_type": "VAEDecode", + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + } + }, + "9": { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + } + } + } + """ + + prompt = json.loads(prompt_text) + #set the text prompt for our positive CLIPTextEncode + prompt["6"]["inputs"]["text"] = "(realistic:1.25), beautiful:1.1) mountain landscape with a deep blue lake, photolike, high detail, monoton colors" + + + seed = 7 + + #set the seed for our KSampler node + prompt["3"]["inputs"]["seed"] = seed + + ws = websocket.WebSocket() + ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id)) + + + + images = get_images(ws, prompt) + + #Commented out code to display the output images: + + for node_id in images: + for image_data in images[node_id]: + from PIL import Image + import io + image = Image.open(io.BytesIO(image_data)) + # image.show() + image.save(f"Output-{seed}.png") diff --git a/test.py b/test.py new file mode 100644 index 00000000..9b50e1dc --- /dev/null +++ b/test.py @@ -0,0 +1,21 @@ +from generation_models import NicheStableDiffusion + + +params = { + "checkpoint_file": "checkpoints/RealisticVision.safetensors", +"download_url": "https://civitai.com/api/download/models/130072?type=Model&format=SafeTensor&size=pruned&fp=fp16", +"scheduler": "dpm++2m", +"supporting_pipelines": ['txt2img'] +} +pipe = NicheStableDiffusion( + **params +) + +input_dict = { + "pipeline_type": "txt2img", + "prompt": "a cat", + "num_inference_steps": 25, +} + +image = pipe(**input_dict) +image.save("debug.webp") \ No newline at end of file From 2519300c0b1f34a11a2f94eabe443db78f7d3d04 Mon Sep 17 00:00:00 2001 From: ThanhDungCR7 Date: Fri, 5 Apr 2024 10:40:27 +0000 Subject: [PATCH 2/2] niche_sticker_maker_comfyui --- generation_models/__init__.py | 3 + generation_models/niche_sticker_maker.py | 52 ++++++-- generation_models/utils_api_comfyui.py | 121 +++--------------- .../{ => workflow-json}/face_to_many.json | 0 .../{ => workflow-json}/sticker_maker.json | 4 +- services/rewarding/app.py | 9 +- test.py | 21 --- 7 files changed, 70 insertions(+), 140 deletions(-) rename generation_models/{ => workflow-json}/face_to_many.json (100%) rename generation_models/{ => workflow-json}/sticker_maker.json (99%) delete mode 100644 test.py diff --git a/generation_models/__init__.py b/generation_models/__init__.py index 9ae66fcf..f4ccece1 100644 --- a/generation_models/__init__.py +++ b/generation_models/__init__.py @@ -1,9 +1,12 @@ from .niche_stable_diffusion import NicheStableDiffusion from .niche_stable_diffusion_xl import NicheStableDiffusionXL from .niche_go_journey import NicheGoJourney +from .niche_sticker_maker import NicheStickerMaker + __all__ = [ "NicheStableDiffusion", "NicheStableDiffusionXL", "NicheGoJourney", + "NicheStickerMaker", ] diff --git a/generation_models/niche_sticker_maker.py b/generation_models/niche_sticker_maker.py index 7986c36d..4ac11407 100644 --- a/generation_models/niche_sticker_maker.py +++ b/generation_models/niche_sticker_maker.py @@ -1,36 +1,68 @@ -from base_model import BaseModel +from .base_model import BaseModel from generation_models.utils_api_comfyui import * +import uuid +from PIL import Image +import io +import base64 + + +server_address = "82.67.70.191:40892" +client_id = "13c08530-8911-4e38-8489-7cded8eddd9d" class NicheStickerMaker(BaseModel): def __init__(self, *args, **kwargs): - self.server_address = "82.67.70.191:40830" - self.client_id = str(uuid.uuid4()) + self.server_address, self.client_id = kwargs.get("server_address"), kwargs.get("client_id") + self.inference_function = self.load_model(*args, **kwargs) + def __call__(self, *args, **kwargs): + return self.inference_function(*args, **kwargs) + def load_model(self, *args, **kwargs): imagine_inference_function = self.load_image(*args, **kwargs) return imagine_inference_function def load_image(self, *args, **kwargs): - workflow = load_workflow("sticker_maker.json") + # workflow = load_workflow("generation_models/sticker_maker.json") def inference_function(*args, **kwargs): + with open("generation_models/workflow-json/sticker_maker.json", "r") as file: + workflow_json = file.read() + + + workflow = json.loads(workflow_json) workflow["2"]["inputs"]["positive"] = kwargs["prompt"] - workflow["4"]["inputs"]["seed"] = seed + workflow["4"]["inputs"]["seed"] = kwargs["seed"] ws = websocket.WebSocket() ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id)) + images = get_images(ws, workflow) + imgs = [] + for node_id in images: + for image_data in images[node_id]: + image = Image.open(io.BytesIO(image_data)) + imgs.append(image) - - images = get_images(ws, prompt) - return images + return imgs[0] return inference_function if __name__=="__main__": - test_sticker = NicheStickerMaker({"prompt":"cute dog", "seed":7}) - test_sticker.load_model() + params = { + "supporting_pipelines": ['txt2img'] + } + pipe = NicheStickerMaker( + **params + ) + + input_dict = { + "pipeline_type": "txt2img", + "prompt": "a cat", + } + + image = pipe(**input_dict) + image.save("debug.webp") diff --git a/generation_models/utils_api_comfyui.py b/generation_models/utils_api_comfyui.py index 334df85c..7f94dc7e 100644 --- a/generation_models/utils_api_comfyui.py +++ b/generation_models/utils_api_comfyui.py @@ -4,8 +4,8 @@ import urllib.request import urllib.parse -server_address = "82.67.70.191:40830" -client_id = str(uuid.uuid4()) +server_address = "82.67.70.191:40892" +client_id = "13c08530-8911-4e38-8489-7cded8eddd9d" def queue_prompt(prompt): p = {"prompt": prompt, "client_id": client_id} @@ -27,7 +27,7 @@ def load_workflow(workflow_path): try: with open(workflow_path, 'r') as file: workflow = json.load(file) - return json.dumps(workflow) + return workflow except FileNotFoundError: print(f"The file {workflow_path} was not found.") return None @@ -40,7 +40,9 @@ def get_images(ws, prompt): output_images = {} while True: out = ws.recv() + if isinstance(out, str): + message = json.loads(out) if message['type'] == 'executing': data = message['data'] @@ -62,118 +64,31 @@ def get_images(ws, prompt): return output_images if __name__ == "__main__": - prompt_text = """ - { - "3": { - "class_type": "KSampler", - "inputs": { - "cfg": 8, - "denoise": 1, - "latent_image": [ - "5", - 0 - ], - "model": [ - "4", - 0 - ], - "negative": [ - "7", - 0 - ], - "positive": [ - "6", - 0 - ], - "sampler_name": "euler", - "scheduler": "normal", - "seed": 8566257, - "steps": 20 - } - }, - "4": { - "class_type": "CheckpointLoaderSimple", - "inputs": { - "ckpt_name": "v1-5-pruned-emaonly.ckpt" - } - }, - "5": { - "class_type": "EmptyLatentImage", - "inputs": { - "batch_size": 1, - "height": 512, - "width": 512 - } - }, - "6": { - "class_type": "CLIPTextEncode", - "inputs": { - "clip": [ - "4", - 1 - ], - "text": "masterpiece best quality girl" - } - }, - "7": { - "class_type": "CLIPTextEncode", - "inputs": { - "clip": [ - "4", - 1 - ], - "text": "bad hands" - } - }, - "8": { - "class_type": "VAEDecode", - "inputs": { - "samples": [ - "3", - 0 - ], - "vae": [ - "4", - 2 - ] - } - }, - "9": { - "class_type": "SaveImage", - "inputs": { - "filename_prefix": "ComfyUI", - "images": [ - "8", - 0 - ] - } - } - } - """ - - prompt = json.loads(prompt_text) - #set the text prompt for our positive CLIPTextEncode - prompt["6"]["inputs"]["text"] = "(realistic:1.25), beautiful:1.1) mountain landscape with a deep blue lake, photolike, high detail, monoton colors" - - seed = 7 + with open("generation_models/workflow-json/sticker_maker.json", "r") as file: + workflow_json = file.read() + - #set the seed for our KSampler node - prompt["3"]["inputs"]["seed"] = seed + workflow = json.loads(workflow_json) + #set the text prompt for our positive CLIPTextEncode + workflow["2"]["inputs"]["positive"] = "a dog" + workflow["4"]["inputs"]["seed"] = 7 ws = websocket.WebSocket() ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id)) + print(ws) + images = get_images(ws, workflow) - images = get_images(ws, prompt) - #Commented out code to display the output images: - + i=0 for node_id in images: for image_data in images[node_id]: + i+=1 from PIL import Image import io image = Image.open(io.BytesIO(image_data)) # image.show() - image.save(f"Output-{seed}.png") + print("out") + image.save(f"Output{i}.webp") diff --git a/generation_models/face_to_many.json b/generation_models/workflow-json/face_to_many.json similarity index 100% rename from generation_models/face_to_many.json rename to generation_models/workflow-json/face_to_many.json diff --git a/generation_models/sticker_maker.json b/generation_models/workflow-json/sticker_maker.json similarity index 99% rename from generation_models/sticker_maker.json rename to generation_models/workflow-json/sticker_maker.json index c116120f..e8bb0be5 100644 --- a/generation_models/sticker_maker.json +++ b/generation_models/workflow-json/sticker_maker.json @@ -28,7 +28,7 @@ "inputs": { "input_mode": "simple", "lora_count": 3, - "lora_name_1": "artificialguybr/StickersRedmond.safetensors", + "lora_name_1": "StickersRedmond.safetensors", "lora_wt_1": 1, "model_str_1": 1, "clip_str_1": 1, @@ -452,4 +452,4 @@ "title": "Save Image" } } -} \ No newline at end of file +} diff --git a/services/rewarding/app.py b/services/rewarding/app.py index 6140c4c0..a4a56e51 100644 --- a/services/rewarding/app.py +++ b/services/rewarding/app.py @@ -136,7 +136,7 @@ async def __call__(self, reward_request: RewardRequest): args = get_args() model_deployment = serve.deployment( ModelDeployment, - name="model_deployment", + name="model_deployment-reward", num_replicas=args.num_replicas, ray_actor_options={"num_gpus": args.num_gpus}, ) @@ -144,12 +144,13 @@ async def __call__(self, reward_request: RewardRequest): model_deployment.bind( MODEL_CONFIG[args.model_name], ), - name="model_deployment", + name="model_deployment-reward", ) - model_handle = serve.get_deployment_handle("model_deployment", "model_deployment") + model_handle = serve.get_deployment_handle("model_deployment-reward", "model_deployment-reward") app = RewardApp(model_handle, args) uvicorn.run( app.app, host="0.0.0.0", port=args.port, - ) \ No newline at end of file + ) + \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100644 index 9b50e1dc..00000000 --- a/test.py +++ /dev/null @@ -1,21 +0,0 @@ -from generation_models import NicheStableDiffusion - - -params = { - "checkpoint_file": "checkpoints/RealisticVision.safetensors", -"download_url": "https://civitai.com/api/download/models/130072?type=Model&format=SafeTensor&size=pruned&fp=fp16", -"scheduler": "dpm++2m", -"supporting_pipelines": ['txt2img'] -} -pipe = NicheStableDiffusion( - **params -) - -input_dict = { - "pipeline_type": "txt2img", - "prompt": "a cat", - "num_inference_steps": 25, -} - -image = pipe(**input_dict) -image.save("debug.webp") \ No newline at end of file