From 0301e53b223fca349c45031eaf8144980e9b41fd Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Sun, 18 Sep 2022 15:05:31 +0900 Subject: [PATCH 01/16] Add an argument "negative_prompt" --- .../pipeline_stable_diffusion.py | 17 ++++++++++++++++- .../pipeline_stable_diffusion_img2img.py | 17 ++++++++++++++++- .../pipeline_stable_diffusion_inpaint.py | 17 ++++++++++++++++- .../pipeline_stable_diffusion_onnx.py | 15 ++++++++++++++- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 9f1211b43013..52c2b91eae5b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -110,6 +110,7 @@ def disable_attention_slicing(self): def __call__( self, prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, @@ -127,6 +128,8 @@ def __call__( Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. width (`int`, *optional*, defaults to 512): @@ -203,9 +206,21 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: + ucond_tokens: List[str] + if negative_prompt is None: + ucond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise ValueError("`negative_prompt` should be the same type to `prompt`.") + elif isinstance(negative_prompt, str): + ucond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + else: + ucond_tokens = negative_prompt + max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens, padding="max_length", max_length=max_length, return_tensors="pt" ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index e7adb4d1a33b..e72a7b731eb2 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -122,6 +122,7 @@ def disable_attention_slicing(self): def __call__( self, prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, init_image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, @@ -137,6 +138,8 @@ def __call__( Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. init_image (`torch.FloatTensor` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. @@ -231,9 +234,21 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: + ucond_tokens: List[str] + if negative_prompt is None: + ucond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise ValueError("`negative_prompt` should be the same type to `prompt`.") + elif isinstance(negative_prompt, str): + ucond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + else: + ucond_tokens = negative_prompt + max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens, padding="max_length", max_length=max_length, return_tensors="pt" ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index b9ad36f1a2bf..5c76ae1524d2 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -141,6 +141,7 @@ def disable_attention_slicing(self): def __call__( self, prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, init_image: Union[torch.FloatTensor, PIL.Image.Image], mask_image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, @@ -157,6 +158,8 @@ def __call__( Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. init_image (`torch.FloatTensor` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. This is the image whose masked region will be inpainted. @@ -264,9 +267,21 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: + ucond_tokens: List[str] + if negative_prompt is None: + ucond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise ValueError("`negative_prompt` should be the same type to `prompt`.") + elif isinstance(negative_prompt, str): + ucond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + else: + ucond_tokens = negative_prompt + max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens * batch_size, padding="max_length", max_length=max_length, return_tensors="pt" ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index ccba29ade5d3..0568933d0d8d 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -45,6 +45,7 @@ def __init__( def __call__( self, prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, @@ -81,9 +82,21 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: + ucond_tokens: List[str] + if negative_prompt is None: + ucond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise ValueError("`negative_prompt` should be the same type to `prompt`.") + elif isinstance(negative_prompt, str): + ucond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + else: + ucond_tokens = negative_prompt + max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="np" + ucond_tokens * batch_size, padding="max_length", max_length=max_length, return_tensors="np" ) uncond_embeddings = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] From 4af6aeb54fc506ed5d3b3ad31e2c3d46a0f04134 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Sun, 18 Sep 2022 15:20:07 +0900 Subject: [PATCH 02/16] Fix argument order --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 6 +++--- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 6 +++--- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 6 +++--- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 52c2b91eae5b..82e8a9667281 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -110,11 +110,11 @@ def disable_attention_slicing(self): def __call__( self, prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, eta: Optional[float] = 0.0, generator: Optional[torch.Generator] = None, latents: Optional[torch.FloatTensor] = None, @@ -128,8 +128,6 @@ def __call__( Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. width (`int`, *optional*, defaults to 512): @@ -143,6 +141,8 @@ def __call__( Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index e72a7b731eb2..a40387594f97 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -122,11 +122,11 @@ def disable_attention_slicing(self): def __call__( self, prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, init_image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, eta: Optional[float] = 0.0, generator: Optional[torch.Generator] = None, output_type: Optional[str] = "pil", @@ -138,8 +138,6 @@ def __call__( Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. init_image (`torch.FloatTensor` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. @@ -158,6 +156,8 @@ def __call__( Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 5c76ae1524d2..1e63275328a3 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -141,12 +141,12 @@ def disable_attention_slicing(self): def __call__( self, prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, init_image: Union[torch.FloatTensor, PIL.Image.Image], mask_image: Union[torch.FloatTensor, PIL.Image.Image], strength: float = 0.8, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, eta: Optional[float] = 0.0, generator: Optional[torch.Generator] = None, output_type: Optional[str] = "pil", @@ -158,8 +158,6 @@ def __call__( Args: prompt (`str` or `List[str]`): The prompt or prompts to guide the image generation. - negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. init_image (`torch.FloatTensor` or `PIL.Image.Image`): `Image`, or tensor representing an image batch, that will be used as the starting point for the process. This is the image whose masked region will be inpainted. @@ -182,6 +180,8 @@ def __call__( Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index 0568933d0d8d..cf08be926ef6 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -45,11 +45,11 @@ def __init__( def __call__( self, prompt: Union[str, List[str]], - negative_prompt: Optional[Union[str, List[str]]] = None, height: Optional[int] = 512, width: Optional[int] = 512, num_inference_steps: Optional[int] = 50, guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, eta: Optional[float] = 0.0, latents: Optional[np.ndarray] = None, output_type: Optional[str] = "pil", From 9c0dbf5d26dc2d1a4fa1a70641fcbc27d6eb186c Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Sun, 18 Sep 2022 23:46:54 +0900 Subject: [PATCH 03/16] Fix to use TypeError instead of ValueError --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 82e8a9667281..4ffe4bb3966c 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -210,7 +210,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise ValueError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] elif batch_size != len(negative_prompt): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index a40387594f97..20bc38320744 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -238,7 +238,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise ValueError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] elif batch_size != len(negative_prompt): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 1e63275328a3..898cacdb22bf 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -271,7 +271,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise ValueError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] elif batch_size != len(negative_prompt): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index cf08be926ef6..c275abf24f04 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -86,7 +86,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise ValueError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] elif batch_size != len(negative_prompt): From 4afb86c6d8e83b4aa9107212349808c1cd31a704 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Sun, 18 Sep 2022 23:51:00 +0900 Subject: [PATCH 04/16] Removed needless batch_size multiplying --- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 898cacdb22bf..3101651183d7 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -281,7 +281,7 @@ def __call__( max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens * batch_size, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens, padding="max_length", max_length=max_length, return_tensors="pt" ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index c275abf24f04..89609dcaf4fb 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -96,7 +96,7 @@ def __call__( max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens * batch_size, padding="max_length", max_length=max_length, return_tensors="np" + ucond_tokens, padding="max_length", max_length=max_length, return_tensors="np" ) uncond_embeddings = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] From 8d8d490ea75ce9ee12266e85ad2a8f78df84b2bf Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Mon, 19 Sep 2022 00:14:16 +0900 Subject: [PATCH 05/16] Fix to multiply by batch_size --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 4ffe4bb3966c..734b40b6c3bb 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -212,7 +212,7 @@ def __call__( elif type(prompt) is not type(negative_prompt): raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] + ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError("The length of `negative_prompt` should be equal to batch_size.") else: diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 20bc38320744..dc143cbd9749 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -240,7 +240,7 @@ def __call__( elif type(prompt) is not type(negative_prompt): raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] + ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError("The length of `negative_prompt` should be equal to batch_size.") else: diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 3101651183d7..e6cc40e00ad4 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -273,7 +273,7 @@ def __call__( elif type(prompt) is not type(negative_prompt): raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] + ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError("The length of `negative_prompt` should be equal to batch_size.") else: diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index 89609dcaf4fb..748b4643a8aa 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -88,7 +88,7 @@ def __call__( elif type(prompt) is not type(negative_prompt): raise TypeError("`negative_prompt` should be the same type to `prompt`.") elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] + ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError("The length of `negative_prompt` should be equal to batch_size.") else: From 0da0e418f29d33b3ad6d3a461bcd77e7a08ba923 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Wed, 21 Sep 2022 00:41:23 +0900 Subject: [PATCH 06/16] Add truncation=True for long negative prompt --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 6 +++++- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 6 +++++- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 6 +++++- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 734b40b6c3bb..b37b24cb94ae 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -220,7 +220,11 @@ def __call__( max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index dc143cbd9749..dca131508915 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -248,7 +248,11 @@ def __call__( max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index e6cc40e00ad4..62e799bf31aa 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -281,7 +281,11 @@ def __call__( max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, padding="max_length", max_length=max_length, return_tensors="pt" + ucond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", ) uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index 748b4643a8aa..2134981b5d33 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -96,7 +96,11 @@ def __call__( max_length = text_input.input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, padding="max_length", max_length=max_length, return_tensors="np" + ucond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", ) uncond_embeddings = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] From 933fda54e433056a86df1943b80caa72a8244092 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:14:29 +0900 Subject: [PATCH 07/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py Co-authored-by: Patrick von Platen --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index fde51bad8dc7..f5c09e7869f8 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -228,7 +228,7 @@ def __call__( elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): - raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + raise ValueError(f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`: {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.") else: ucond_tokens = negative_prompt From d594ee819746c41cda06786b2182927060a1d19e Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:14:42 +0900 Subject: [PATCH 08/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py Co-authored-by: Patrick von Platen --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index f5c09e7869f8..6507d8c62f27 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -224,7 +224,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): From 43ac1d39ae5af6aa0e0089cf7a7df17d22b1e548 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:15:00 +0900 Subject: [PATCH 09/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py Co-authored-by: Patrick von Platen --- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index e02c838ca7f0..81dc85010566 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -303,7 +303,7 @@ def __call__( elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): - raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + raise ValueError(f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`: {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.") else: ucond_tokens = negative_prompt From 2cc75d329c0f55aee1aca1af28a574ba99f4701d Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:15:10 +0900 Subject: [PATCH 10/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py Co-authored-by: Patrick von Platen --- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index 0517b0e3406f..98ab87f4bee3 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -111,7 +111,7 @@ def __call__( elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): - raise ValueError("The length of `negative_prompt` should be equal to batch_size.") + raise ValueError(f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`: {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.") else: ucond_tokens = negative_prompt From 004d68fdfe5febd0d3c5d105f289d4a56e43e91b Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:15:22 +0900 Subject: [PATCH 11/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py Co-authored-by: Patrick von Platen --- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 7234af2dcec9..debc4a774aa1 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -265,7 +265,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): From 9d2e7511e64ce89779c07ca78d56b9aee437a478 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:15:35 +0900 Subject: [PATCH 12/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py Co-authored-by: Patrick von Platen --- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 81dc85010566..e587e96c851b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -299,7 +299,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): From 02fc05891b2967b6252af9d0c0df219222ece680 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:15:47 +0900 Subject: [PATCH 13/16] Update src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py Co-authored-by: Patrick von Platen --- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index 98ab87f4bee3..a36b7f89c05f 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -107,7 +107,7 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`.") + raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): From e875fc4d1ce4e7aa3a5972f4aec7bc3e40c70ac3 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:16:30 +0900 Subject: [PATCH 14/16] Fix styles --- .../stable_diffusion/pipeline_stable_diffusion.py | 11 +++++++++-- .../pipeline_stable_diffusion_img2img.py | 5 ++++- .../pipeline_stable_diffusion_inpaint.py | 11 +++++++++-- .../pipeline_stable_diffusion_onnx.py | 11 +++++++++-- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 6507d8c62f27..44a354ee0474 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -224,11 +224,18 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") + raise TypeError( + "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + " {type(prompt)}." + ) elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): - raise ValueError(f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`: {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.") + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) else: ucond_tokens = negative_prompt diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index debc4a774aa1..83a547246c23 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -265,7 +265,10 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") + raise TypeError( + "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + " {type(prompt)}." + ) elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index e587e96c851b..26a43b9305c2 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -299,11 +299,18 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") + raise TypeError( + "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + " {type(prompt)}." + ) elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): - raise ValueError(f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`: {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.") + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) else: ucond_tokens = negative_prompt diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index a36b7f89c05f..a880efd4e50a 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -107,11 +107,18 @@ def __call__( if negative_prompt is None: ucond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): - raise TypeError("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} != {type(prompt)}.") + raise TypeError( + "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + " {type(prompt)}." + ) elif isinstance(negative_prompt, str): ucond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): - raise ValueError(f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`: {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches the batch size of `prompt`.") + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) else: ucond_tokens = negative_prompt From 4c442e69f6afa9a24038736324a80bf031185169 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:39:14 +0900 Subject: [PATCH 15/16] Renamed ucond_tokens to uncond_tokens --- .../stable_diffusion/pipeline_stable_diffusion.py | 10 +++++----- .../pipeline_stable_diffusion_img2img.py | 10 +++++----- .../pipeline_stable_diffusion_inpaint.py | 10 +++++----- .../stable_diffusion/pipeline_stable_diffusion_onnx.py | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 44a354ee0474..91f164f81f33 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -220,16 +220,16 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - ucond_tokens: List[str] + uncond_tokens: List[str] if negative_prompt is None: - ucond_tokens = [""] * batch_size + uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): raise TypeError( "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" " {type(prompt)}." ) elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] * batch_size + uncond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError( f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" @@ -237,11 +237,11 @@ def __call__( " the batch size of `prompt`." ) else: - ucond_tokens = negative_prompt + uncond_tokens = negative_prompt max_length = text_input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, + uncond_tokens, padding="max_length", max_length=max_length, truncation=True, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 83a547246c23..aa184b11a147 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -261,24 +261,24 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - ucond_tokens: List[str] + uncond_tokens: List[str] if negative_prompt is None: - ucond_tokens = [""] * batch_size + uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): raise TypeError( "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" " {type(prompt)}." ) elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] * batch_size + uncond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError("The length of `negative_prompt` should be equal to batch_size.") else: - ucond_tokens = negative_prompt + uncond_tokens = negative_prompt max_length = text_input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, + uncond_tokens, padding="max_length", max_length=max_length, truncation=True, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 26a43b9305c2..a4a817275c78 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -295,16 +295,16 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - ucond_tokens: List[str] + uncond_tokens: List[str] if negative_prompt is None: - ucond_tokens = [""] * batch_size + uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): raise TypeError( "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" " {type(prompt)}." ) elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] * batch_size + uncond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError( f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" @@ -312,11 +312,11 @@ def __call__( " the batch size of `prompt`." ) else: - ucond_tokens = negative_prompt + uncond_tokens = negative_prompt max_length = text_input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, + uncond_tokens, padding="max_length", max_length=max_length, truncation=True, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py index a880efd4e50a..0bb0ce440d90 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_onnx.py @@ -103,16 +103,16 @@ def __call__( do_classifier_free_guidance = guidance_scale > 1.0 # get unconditional embeddings for classifier free guidance if do_classifier_free_guidance: - ucond_tokens: List[str] + uncond_tokens: List[str] if negative_prompt is None: - ucond_tokens = [""] * batch_size + uncond_tokens = [""] * batch_size elif type(prompt) is not type(negative_prompt): raise TypeError( "`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" " {type(prompt)}." ) elif isinstance(negative_prompt, str): - ucond_tokens = [negative_prompt] * batch_size + uncond_tokens = [negative_prompt] * batch_size elif batch_size != len(negative_prompt): raise ValueError( f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" @@ -120,11 +120,11 @@ def __call__( " the batch size of `prompt`." ) else: - ucond_tokens = negative_prompt + uncond_tokens = negative_prompt max_length = text_input_ids.shape[-1] uncond_input = self.tokenizer( - ucond_tokens, + uncond_tokens, padding="max_length", max_length=max_length, truncation=True, From 3afe0aa14335875a31df0d6e978258664ac8b9ad Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 4 Oct 2022 20:46:04 +0900 Subject: [PATCH 16/16] Added description about "negative_prompt" --- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 3 ++- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 3 ++- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 91f164f81f33..bc6ca1efbd9b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -146,7 +146,8 @@ def __call__( 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index aa184b11a147..f0846573d34a 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -162,7 +162,8 @@ def __call__( 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others. diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index a4a817275c78..107c4de07197 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -182,7 +182,8 @@ def __call__( 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality. negative_prompt (`str` or `List[str]`, *optional*): - The prompt or prompts not to guide the image generation. + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). eta (`float`, *optional*, defaults to 0.0): Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to [`schedulers.DDIMScheduler`], will be ignored for others.