import{S as sn,i as rn,s as ln,e as n,k as l,w as y,t as s,M as dn,c as i,d as o,m as d,a,x as w,h as r,b as p,N as pn,G as e,g,y as x,q as C,o as T,B as L,v as cn,L as mn}from"../../../chunks/vendor-hf-doc-builder.js";import{D as B}from"../../../chunks/Docstring-hf-doc-builder.js";import{C as fn}from"../../../chunks/CodeBlock-hf-doc-builder.js";import{I as an}from"../../../chunks/IconCopyLink-hf-doc-builder.js";import{E as gn}from"../../../chunks/ExampleCodeBlock-hf-doc-builder.js";function un($e){let h,k,b,_,N;return _=new fn({props:{code:"JTIzJTIwIXBpcCUyMGluc3RhbGwlMjBvcGVuY3YtcHl0aG9uJTIwdHJhbnNmb3JtZXJzJTIwYWNjZWxlcmF0ZSUwQWZyb20lMjBkaWZmdXNlcnMlMjBpbXBvcnQlMjBTdGFibGVEaWZmdXNpb25YTENvbnRyb2xOZXRQaXBlbGluZSUyQyUyMENvbnRyb2xOZXRNb2RlbCUyQyUyMEF1dG9lbmNvZGVyS0wlMEFmcm9tJTIwZGlmZnVzZXJzLnV0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQWltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBaW1wb3J0JTIwY3YyJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIyYWVyaWFsJTIwdmlldyUyQyUyMGElMjBmdXR1cmlzdGljJTIwcmVzZWFyY2glMjBjb21wbGV4JTIwaW4lMjBhJTIwYnJpZ2h0JTIwZm9nZ3klMjBqdW5nbGUlMkMlMjBoYXJkJTIwbGlnaHRpbmclMjIlMEFuZWdhdGl2ZV9wcm9tcHQlMjAlM0QlMjAlMjJsb3clMjBxdWFsaXR5JTJDJTIwYmFkJTIwcXVhbGl0eSUyQyUyMHNrZXRjaGVzJTIyJTBBJTBBJTIzJTIwZG93bmxvYWQlMjBhbiUyMGltYWdlJTBBaW1hZ2UlMjAlM0QlMjBsb2FkX2ltYWdlKCUwQSUyMCUyMCUyMCUyMCUyMmh0dHBzJTNBJTJGJTJGaGYuY28lMkZkYXRhc2V0cyUyRmhmLWludGVybmFsLXRlc3RpbmclMkZkaWZmdXNlcnMtaW1hZ2VzJTJGcmVzb2x2ZSUyRm1haW4lMkZzZF9jb250cm9sbmV0JTJGaGYtbG9nby5wbmclMjIlMEEpJTBBJTBBJTIzJTIwaW5pdGlhbGl6ZSUyMHRoZSUyMG1vZGVscyUyMGFuZCUyMHBpcGVsaW5lJTBBY29udHJvbG5ldF9jb25kaXRpb25pbmdfc2NhbGUlMjAlM0QlMjAwLjUlMjAlMjAlMjMlMjByZWNvbW1lbmRlZCUyMGZvciUyMGdvb2QlMjBnZW5lcmFsaXphdGlvbiUwQWNvbnRyb2xuZXQlMjAlM0QlMjBDb250cm9sTmV0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpZmZ1c2VycyUyRmNvbnRyb2xuZXQtY2Fubnktc2R4bC0xLjAlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMEEpJTBBdmFlJTIwJTNEJTIwQXV0b2VuY29kZXJLTC5mcm9tX3ByZXRyYWluZWQoJTIybWFkZWJ5b2xsaW4lMkZzZHhsLXZhZS1mcDE2LWZpeCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uWExDb250cm9sTmV0UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi14bC1iYXNlLTEuMCUyMiUyQyUyMGNvbnRyb2xuZXQlM0Rjb250cm9sbmV0JTJDJTIwdmFlJTNEdmFlJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTBBKSUwQXBpcGUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCklMEElMEElMjMlMjBnZXQlMjBjYW5ueSUyMGltYWdlJTBBaW1hZ2UlMjAlM0QlMjBucC5hcnJheShpbWFnZSklMEFpbWFnZSUyMCUzRCUyMGN2Mi5DYW5ueShpbWFnZSUyQyUyMDEwMCUyQyUyMDIwMCklMEFpbWFnZSUyMCUzRCUyMGltYWdlJTVCJTNBJTJDJTIwJTNBJTJDJTIwTm9uZSU1RCUwQWltYWdlJTIwJTNEJTIwbnAuY29uY2F0ZW5hdGUoJTVCaW1hZ2UlMkMlMjBpbWFnZSUyQyUyMGltYWdlJTVEJTJDJTIwYXhpcyUzRDIpJTBBY2FubnlfaW1hZ2UlMjAlM0QlMjBJbWFnZS5mcm9tYXJyYXkoaW1hZ2UpJTBBJTBBJTIzJTIwZ2VuZXJhdGUlMjBpbWFnZSUwQWltYWdlJTIwJTNEJTIwcGlwZSglMEElMjAlMjAlMjAlMjBwcm9tcHQlMkMlMjBjb250cm9sbmV0X2NvbmRpdGlvbmluZ19zY2FsZSUzRGNvbnRyb2xuZXRfY29uZGl0aW9uaW5nX3NjYWxlJTJDJTIwaW1hZ2UlM0RjYW5ueV9pbWFnZSUwQSkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># !pip install opencv-python transformers accelerate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> cv2
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;aerial view, a futuristic research complex in a bright foggy jungle, hard lighting&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>negative_prompt = <span class="hljs-string">&quot;low quality, bad quality, sketches&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># download an image</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = load_image(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;https://hf.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize the models and pipeline</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>controlnet_conditioning_scale = <span class="hljs-number">0.5</span>  <span class="hljs-comment"># recommended for good generalization</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>controlnet = ControlNetModel.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;diffusers/controlnet-canny-sdxl-1.0&quot;</span>, torch_dtype=torch.float16
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, controlnet=controlnet, vae=vae, torch_dtype=torch.float16
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe.enable_model_cpu_offload()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># get canny image</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = np.array(image)
<span class="hljs-meta">&gt;&gt;&gt; </span>image = cv2.Canny(image, <span class="hljs-number">100</span>, <span class="hljs-number">200</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>image = image[:, :, <span class="hljs-literal">None</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>canny_image = Image.fromarray(image)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># generate image</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = pipe(
<span class="hljs-meta">... </span>    prompt, controlnet_conditioning_scale=controlnet_conditioning_scale, image=canny_image
<span class="hljs-meta">... </span>).images[<span class="hljs-number">0</span>]`}}),{c(){h=n("p"),k=s("Examples:"),b=l(),y(_.$$.fragment)},l(u){h=i(u,"P",{});var M=a(h);k=r(M,"Examples:"),M.forEach(o),b=d(u),w(_.$$.fragment,u)},m(u,M){g(u,h,M),e(h,k),g(u,b,M),x(_,u,M),N=!0},p:mn,i(u){N||(C(_.$$.fragment,u),N=!0)},o(u){T(_.$$.fragment,u),N=!1},d(u){u&&o(h),u&&o(b),L(_,u)}}}function hn($e){let h,k,b,_,N,u,M,ye,pt,Ze,F,z,ct,mt,We,pe,ft,Ge,ce,gt,Be,me,we,ut,Fe,X,ht,fe,_t,bt,ze,D,vt,A,yt,wt,V,xt,Ct,Ae,ge,Tt,Ve,R,Lo,Re,S,j,xe,Y,Lt,Ce,Nt,Ye,c,Q,Mt,Te,Dt,Pt,O,It,ue,St,kt,Xt,Le,jt,Et,q,H,Ne,Jt,Ut,he,$t,Zt,K,Me,Wt,Gt,_e,Bt,Ft,P,ee,zt,De,At,Vt,E,Rt,J,te,Yt,oe,Qt,Pe,Ot,qt,Ht,U,ne,Kt,ie,eo,Ie,to,oo,no,$,ae,io,v,ao,Se,so,ro,ke,lo,po,Xe,co,mo,je,fo,go,uo,Z,se,ho,Ee,_o,bo,W,re,vo,Je,yo,wo,G,le,xo,Ue,Co,Qe;return u=new an({}),Y=new an({}),Q=new B({props:{name:"class diffusers.StableDiffusionXLControlNetPipeline",anchor:"diffusers.StableDiffusionXLControlNetPipeline",parameters:[{name:"vae",val:": AutoencoderKL"},{name:"text_encoder",val:": CLIPTextModel"},{name:"text_encoder_2",val:": CLIPTextModelWithProjection"},{name:"tokenizer",val:": CLIPTokenizer"},{name:"tokenizer_2",val:": CLIPTokenizer"},{name:"unet",val:": UNet2DConditionModel"},{name:"controlnet",val:": ControlNetModel"},{name:"scheduler",val:": KarrasDiffusionSchedulers"},{name:"force_zeros_for_empty_prompt",val:": bool = True"},{name:"add_watermarker",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"diffusers.StableDiffusionXLControlNetPipeline.vae",description:`<strong>vae</strong> (<a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a>) &#x2014;
Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.`,name:"vae"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.text_encoder",description:`<strong>text_encoder</strong> (<code>CLIPTextModel</code>) &#x2014;
Frozen text-encoder. Stable Diffusion uses the text portion of
<a href="https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel" rel="nofollow">CLIP</a>, specifically
the <a href="https://huggingface.co/openai/clip-vit-large-patch14" rel="nofollow">clip-vit-large-patch14</a> variant.`,name:"text_encoder"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.text_encoder_2",description:`<strong>text_encoder_2</strong> (<code> CLIPTextModelWithProjection</code>) &#x2014;
Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
<a href="https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection" rel="nofollow">CLIP</a>,
specifically the
<a href="https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k" rel="nofollow">laion/CLIP-ViT-bigG-14-laion2B-39B-b160k</a>
variant.`,name:"text_encoder_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.tokenizer",description:`<strong>tokenizer</strong> (<code>CLIPTokenizer</code>) &#x2014;
Tokenizer of class
<a href="https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer" rel="nofollow">CLIPTokenizer</a>.`,name:"tokenizer"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.tokenizer_2",description:`<strong>tokenizer_2</strong> (<code>CLIPTokenizer</code>) &#x2014;
Second Tokenizer of class
<a href="https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer" rel="nofollow">CLIPTokenizer</a>.`,name:"tokenizer_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.unet",description:'<strong>unet</strong> (<a href="/docs/diffusers/main/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel">UNet2DConditionModel</a>) &#x2014; Conditional U-Net architecture to denoise the encoded image latents.',name:"unet"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.controlnet",description:`<strong>controlnet</strong> (<a href="/docs/diffusers/main/en/api/models/controlnet#diffusers.ControlNetModel">ControlNetModel</a> or <code>List[ControlNetModel]</code>) &#x2014;
Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets
as a list, the outputs from each ControlNet are added together to create one combined additional
conditioning.`,name:"controlnet"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.scheduler",description:`<strong>scheduler</strong> (<a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.SchedulerMixin">SchedulerMixin</a>) &#x2014;
A scheduler to be used in combination with <code>unet</code> to denoise the encoded image latents. Can be one of
<a href="/docs/diffusers/main/en/api/schedulers/ddim#diffusers.DDIMScheduler">DDIMScheduler</a>, <a href="/docs/diffusers/main/en/api/schedulers/lms_discrete#diffusers.LMSDiscreteScheduler">LMSDiscreteScheduler</a>, or <a href="/docs/diffusers/main/en/api/schedulers/pndm#diffusers.PNDMScheduler">PNDMScheduler</a>.`,name:"scheduler"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L104"}}),ee=new B({props:{name:"__call__",anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__",parameters:[{name:"prompt",val:": typing.Union[str, typing.List[str]] = None"},{name:"prompt_2",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"image",val:": typing.Union[torch.FloatTensor, PIL.Image.Image, numpy.ndarray, typing.List[torch.FloatTensor], typing.List[PIL.Image.Image], typing.List[numpy.ndarray]] = None"},{name:"height",val:": typing.Optional[int] = None"},{name:"width",val:": typing.Optional[int] = None"},{name:"num_inference_steps",val:": int = 50"},{name:"guidance_scale",val:": float = 5.0"},{name:"negative_prompt",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"negative_prompt_2",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"num_images_per_prompt",val:": typing.Optional[int] = 1"},{name:"eta",val:": float = 0.0"},{name:"generator",val:": typing.Union[torch._C.Generator, typing.List[torch._C.Generator], NoneType] = None"},{name:"latents",val:": typing.Optional[torch.FloatTensor] = None"},{name:"prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_type",val:": typing.Optional[str] = 'pil'"},{name:"return_dict",val:": bool = True"},{name:"callback",val:": typing.Union[typing.Callable[[int, int, torch.FloatTensor], NoneType], NoneType] = None"},{name:"callback_steps",val:": int = 1"},{name:"cross_attention_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"controlnet_conditioning_scale",val:": typing.Union[float, typing.List[float]] = 1.0"},{name:"guess_mode",val:": bool = False"},{name:"control_guidance_start",val:": typing.Union[float, typing.List[float]] = 0.0"},{name:"control_guidance_end",val:": typing.Union[float, typing.List[float]] = 1.0"},{name:"original_size",val:": typing.Tuple[int, int] = None"},{name:"crops_coords_top_left",val:": typing.Tuple[int, int] = (0, 0)"},{name:"target_size",val:": typing.Tuple[int, int] = None"}],parametersDescription:[{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.prompt",description:`<strong>prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to guide the image generation. If not defined, one has to pass <code>prompt_embeds</code>.
instead.`,name:"prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.prompt_2",description:`<strong>prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to be sent to the <code>tokenizer_2</code> and <code>text_encoder_2</code>. If not defined, <code>prompt</code> is
used in both text-encoders`,name:"prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.image",description:`<strong>image</strong> (<code>torch.FloatTensor</code>, <code>PIL.Image.Image</code>, <code>np.ndarray</code>, <code>List[torch.FloatTensor]</code>, <code>List[PIL.Image.Image]</code>, <code>List[np.ndarray]</code>, &#x2014;
<code>List[List[torch.FloatTensor]]</code>, <code>List[List[np.ndarray]]</code> or <code>List[List[PIL.Image.Image]]</code>):
The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If
the type is specified as <code>Torch.FloatTensor</code>, it is passed to ControlNet as is. <code>PIL.Image.Image</code> can
also be accepted as an image. The dimensions of the output image defaults to <code>image</code>&#x2019;s dimensions. If
height and/or width are passed, <code>image</code> is resized according to them. If multiple ControlNets are
specified in init, images must be passed as a list such that each element of the list can be correctly
batched for input to a single controlnet.`,name:"image"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.height",description:`<strong>height</strong> (<code>int</code>, <em>optional</em>, defaults to self.unet.config.sample_size * self.vae_scale_factor) &#x2014;
The height in pixels of the generated image.`,name:"height"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.width",description:`<strong>width</strong> (<code>int</code>, <em>optional</em>, defaults to self.unet.config.sample_size * self.vae_scale_factor) &#x2014;
The width in pixels of the generated image.`,name:"width"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.num_inference_steps",description:`<strong>num_inference_steps</strong> (<code>int</code>, <em>optional</em>, defaults to 50) &#x2014;
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
expense of slower inference.`,name:"num_inference_steps"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.guidance_scale",description:`<strong>guidance_scale</strong> (<code>float</code>, <em>optional</em>, defaults to 7.5) &#x2014;
Guidance scale as defined in <a href="https://arxiv.org/abs/2207.12598" rel="nofollow">Classifier-Free Diffusion Guidance</a>.
<code>guidance_scale</code> is defined as <code>w</code> of equation 2. of <a href="https://arxiv.org/pdf/2205.11487.pdf" rel="nofollow">Imagen
Paper</a>. Guidance scale is enabled by setting <code>guidance_scale &gt; 1</code>. Higher guidance scale encourages to generate images that are closely linked to the text <code>prompt</code>,
usually at the expense of lower image quality.`,name:"guidance_scale"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_prompt",description:`<strong>negative_prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation. If not defined, one has to pass
<code>negative_prompt_embeds</code> instead. Ignored when not using guidance (i.e., ignored if <code>guidance_scale</code> is
less than <code>1</code>).`,name:"negative_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_prompt_2",description:`<strong>negative_prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation to be sent to <code>tokenizer_2</code> and
<code>text_encoder_2</code>. If not defined, <code>negative_prompt</code> is used in both text-encoders`,name:"negative_prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.num_images_per_prompt",description:`<strong>num_images_per_prompt</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of images to generate per prompt.`,name:"num_images_per_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.eta",description:`<strong>eta</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Corresponds to parameter eta (&#x3B7;) in the DDIM paper: <a href="https://arxiv.org/abs/2010.02502" rel="nofollow">https://arxiv.org/abs/2010.02502</a>. Only applies to
<a href="/docs/diffusers/main/en/api/schedulers/ddim#diffusers.DDIMScheduler">schedulers.DDIMScheduler</a>, will be ignored for others.`,name:"eta"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.generator",description:`<strong>generator</strong> (<code>torch.Generator</code> or <code>List[torch.Generator]</code>, <em>optional</em>) &#x2014;
One or a list of <a href="https://pytorch.org/docs/stable/generated/torch.Generator.html" rel="nofollow">torch generator(s)</a>
to make generation deterministic.`,name:"generator"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.latents",description:`<strong>latents</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
tensor will ge generated by sampling using the supplied random <code>generator</code>.`,name:"latents"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.prompt_embeds",description:`<strong>prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting. If not
provided, text embeddings will be generated from <code>prompt</code> input argument.`,name:"prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_prompt_embeds",description:`<strong>negative_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, negative_prompt_embeds will be generated from <code>negative_prompt</code> input
argument.`,name:"negative_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.pooled_prompt_embeds",description:`<strong>pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting.
If not provided, pooled text embeddings will be generated from <code>prompt</code> input argument.`,name:"pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.negative_pooled_prompt_embeds",description:`<strong>negative_pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, pooled negative_prompt_embeds will be generated from <code>negative_prompt</code>
input argument.`,name:"negative_pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.output_type",description:`<strong>output_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pil&quot;</code>) &#x2014;
The output format of the generate image. Choose between
<a href="https://pillow.readthedocs.io/en/stable/" rel="nofollow">PIL</a>: <code>PIL.Image.Image</code> or <code>np.array</code>.`,name:"output_type"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to return a <a href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/upscale#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput">StableDiffusionPipelineOutput</a> instead of a
plain tuple.`,name:"return_dict"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.callback",description:`<strong>callback</strong> (<code>Callable</code>, <em>optional</em>) &#x2014;
A function that will be called every <code>callback_steps</code> steps during inference. The function will be
called with the following arguments: <code>callback(step: int, timestep: int, latents: torch.FloatTensor)</code>.`,name:"callback"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.callback_steps",description:`<strong>callback_steps</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The frequency at which the <code>callback</code> function will be called. If not specified, the callback will be
called at every step.`,name:"callback_steps"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.cross_attention_kwargs",description:`<strong>cross_attention_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
A kwargs dictionary that if specified is passed along to the <code>AttentionProcessor</code> as defined under
<code>self.processor</code> in
<a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py" rel="nofollow">diffusers.models.attention_processor</a>.`,name:"cross_attention_kwargs"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.controlnet_conditioning_scale",description:`<strong>controlnet_conditioning_scale</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The outputs of the controlnet are multiplied by <code>controlnet_conditioning_scale</code> before they are added
to the residual in the original unet. If multiple ControlNets are specified in init, you can set the
corresponding scale as a list.`,name:"controlnet_conditioning_scale"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.guess_mode",description:`<strong>guess_mode</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
In this mode, the ControlNet encoder will try best to recognize the content of the input image even if
you remove all prompts. The <code>guidance_scale</code> between 3.0 and 5.0 is recommended.`,name:"guess_mode"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.control_guidance_start",description:`<strong>control_guidance_start</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The percentage of total steps at which the controlnet starts applying.`,name:"control_guidance_start"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.control_guidance_end",description:`<strong>control_guidance_end</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The percentage of total steps at which the controlnet stops applying.`,name:"control_guidance_end"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.original_size",description:`<strong>original_size</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to (1024, 1024)) &#x2014;
If <code>original_size</code> is not the same as <code>target_size</code> the image will appear to be down- or upsampled.
<code>original_size</code> defaults to <code>(width, height)</code> if not specified. Part of SDXL&#x2019;s micro-conditioning as
explained in section 2.2 of
<a href="https://huggingface.co/papers/2307.01952" rel="nofollow">https://huggingface.co/papers/2307.01952</a>.`,name:"original_size"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.crops_coords_top_left",description:`<strong>crops_coords_top_left</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to (0, 0)) &#x2014;
<code>crops_coords_top_left</code> can be used to generate an image that appears to be &#x201C;cropped&#x201D; from the position
<code>crops_coords_top_left</code> downwards. Favorable, well-centered images are usually achieved by setting
<code>crops_coords_top_left</code> to (0, 0). Part of SDXL&#x2019;s micro-conditioning as explained in section 2.2 of
<a href="https://huggingface.co/papers/2307.01952" rel="nofollow">https://huggingface.co/papers/2307.01952</a>.`,name:"crops_coords_top_left"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.target_size",description:`<strong>target_size</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to (1024, 1024)) &#x2014;
For most cases, <code>target_size</code> should be set to the desired height and width of the generated image. If
not specified it will default to <code>(width, height)</code>. Part of SDXL&#x2019;s micro-conditioning as explained in
section 2.2 of <a href="https://huggingface.co/papers/2307.01952" rel="nofollow">https://huggingface.co/papers/2307.01952</a>.`,name:"target_size"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L694",returnDescription:`
<p><a
  href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/upscale#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput"
>StableDiffusionPipelineOutput</a> if <code>return_dict</code> is True, otherwise a <code>tuple</code>
containing the output images.</p>
`,returnType:`
<p><a
  href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/upscale#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput"
>StableDiffusionPipelineOutput</a> or <code>tuple</code></p>
`}}),E=new gn({props:{anchor:"diffusers.StableDiffusionXLControlNetPipeline.__call__.example",$$slots:{default:[un]},$$scope:{ctx:$e}}}),te=new B({props:{name:"disable_vae_slicing",anchor:"diffusers.StableDiffusionXLControlNetPipeline.disable_vae_slicing",parameters:[],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L195"}}),ne=new B({props:{name:"disable_vae_tiling",anchor:"diffusers.StableDiffusionXLControlNetPipeline.disable_vae_tiling",parameters:[],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L212"}}),ae=new B({props:{name:"enable_model_cpu_offload",anchor:"diffusers.StableDiffusionXLControlNetPipeline.enable_model_cpu_offload",parameters:[{name:"gpu_id",val:" = 0"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L219"}}),se=new B({props:{name:"enable_vae_slicing",anchor:"diffusers.StableDiffusionXLControlNetPipeline.enable_vae_slicing",parameters:[],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L187"}}),re=new B({props:{name:"enable_vae_tiling",anchor:"diffusers.StableDiffusionXLControlNetPipeline.enable_vae_tiling",parameters:[],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L203"}}),le=new B({props:{name:"encode_prompt",anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt",parameters:[{name:"prompt",val:": str"},{name:"prompt_2",val:": typing.Optional[str] = None"},{name:"device",val:": typing.Optional[torch.device] = None"},{name:"num_images_per_prompt",val:": int = 1"},{name:"do_classifier_free_guidance",val:": bool = True"},{name:"negative_prompt",val:": typing.Optional[str] = None"},{name:"negative_prompt_2",val:": typing.Optional[str] = None"},{name:"prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_pooled_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"lora_scale",val:": typing.Optional[float] = None"}],parametersDescription:[{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.prompt",description:`<strong>prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
prompt to be encoded`,name:"prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.prompt_2",description:`<strong>prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to be sent to the <code>tokenizer_2</code> and <code>text_encoder_2</code>. If not defined, <code>prompt</code> is
used in both text-encoders
device &#x2014; (<code>torch.device</code>):
torch device`,name:"prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.num_images_per_prompt",description:`<strong>num_images_per_prompt</strong> (<code>int</code>) &#x2014;
number of images that should be generated per prompt`,name:"num_images_per_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.do_classifier_free_guidance",description:`<strong>do_classifier_free_guidance</strong> (<code>bool</code>) &#x2014;
whether to use classifier free guidance or not`,name:"do_classifier_free_guidance"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_prompt",description:`<strong>negative_prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation. If not defined, one has to pass
<code>negative_prompt_embeds</code> instead. Ignored when not using guidance (i.e., ignored if <code>guidance_scale</code> is
less than <code>1</code>).`,name:"negative_prompt"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_prompt_2",description:`<strong>negative_prompt_2</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the image generation to be sent to <code>tokenizer_2</code> and
<code>text_encoder_2</code>. If not defined, <code>negative_prompt</code> is used in both text-encoders`,name:"negative_prompt_2"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.prompt_embeds",description:`<strong>prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting. If not
provided, text embeddings will be generated from <code>prompt</code> input argument.`,name:"prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_prompt_embeds",description:`<strong>negative_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, negative_prompt_embeds will be generated from <code>negative_prompt</code> input
argument.`,name:"negative_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.pooled_prompt_embeds",description:`<strong>pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt weighting.
If not provided, pooled text embeddings will be generated from <code>prompt</code> input argument.`,name:"pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.negative_pooled_prompt_embeds",description:`<strong>negative_pooled_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, <em>e.g.</em> prompt
weighting. If not provided, pooled negative_prompt_embeds will be generated from <code>negative_prompt</code>
input argument.`,name:"negative_pooled_prompt_embeds"},{anchor:"diffusers.StableDiffusionXLControlNetPipeline.encode_prompt.lora_scale",description:`<strong>lora_scale</strong> (<code>float</code>, <em>optional</em>) &#x2014;
A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.`,name:"lora_scale"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py#L252"}}),{c(){h=n("meta"),k=l(),b=n("h1"),_=n("a"),N=n("span"),y(u.$$.fragment),M=l(),ye=n("span"),pt=s("ControlNet with Stable Diffusion XL"),Ze=l(),F=n("p"),z=n("a"),ct=s("Adding Conditional Control to Text-to-Image Diffusion Models"),mt=s(" by Lvmin Zhang and Maneesh Agrawala."),We=l(),pe=n("p"),ft=s("Using a pretrained model, we can provide control images (for example, a depth map) to control Stable Diffusion text-to-image generation so that it follows the structure of the depth image and fills in the details."),Ge=l(),ce=n("p"),gt=s("The abstract from the paper is:"),Be=l(),me=n("p"),we=n("em"),ut=s("We present a neural network structure, ControlNet, to control pretrained large diffusion models to support additional input conditions. The ControlNet learns task-specific conditions in an end-to-end way, and the learning is robust even when the training dataset is small (< 50k). Moreover, training a ControlNet is as fast as fine-tuning a diffusion model, and the model can be trained on a personal devices. Alternatively, if powerful computation clusters are available, the model can scale to large amounts (millions to billions) of data. We report that large diffusion models like Stable Diffusion can be augmented with ControlNets to enable conditional inputs like edge maps, segmentation maps, keypoints, etc. This may enrich the methods to control large diffusion models and further facilitate related applications."),Fe=l(),X=n("p"),ht=s("We provide support using ControlNets with "),fe=n("a"),_t=s("Stable Diffusion XL"),bt=s(" (SDXL)."),ze=l(),D=n("p"),vt=s("There are not many ControlNet checkpoints that are compatible with SDXL at the moment. So, we trained one using Canny edge maps as the conditioning images. To know more, check out the "),A=n("a"),yt=s("model card"),wt=s(". We encourage you to train custom ControlNets; we provide a "),V=n("a"),xt=s("training script"),Ct=s(" for this."),Ae=l(),ge=n("p"),Tt=s("You can find some results below:"),Ve=l(),R=n("img"),Re=l(),S=n("h2"),j=n("a"),xe=n("span"),y(Y.$$.fragment),Lt=l(),Ce=n("span"),Nt=s("StableDiffusionXLControlNetPipeline"),Ye=l(),c=n("div"),y(Q.$$.fragment),Mt=l(),Te=n("p"),Dt=s("Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance."),Pt=l(),O=n("p"),It=s("This model inherits from "),ue=n("a"),St=s("DiffusionPipeline"),kt=s(`. Check the superclass documentation for the generic methods the
library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)`),Xt=l(),Le=n("p"),jt=s("In addition the pipeline inherits the following loading methods:"),Et=l(),q=n("ul"),H=n("li"),Ne=n("em"),Jt=s("Textual-Inversion"),Ut=s(": "),he=n("a"),$t=s("loaders.TextualInversionLoaderMixin.load_textual_inversion()"),Zt=l(),K=n("li"),Me=n("em"),Wt=s("LoRA"),Gt=s(": "),_e=n("a"),Bt=s("loaders.LoraLoaderMixin.load_lora_weights()"),Ft=l(),P=n("div"),y(ee.$$.fragment),zt=l(),De=n("p"),At=s("Function invoked when calling the pipeline for generation."),Vt=l(),y(E.$$.fragment),Rt=l(),J=n("div"),y(te.$$.fragment),Yt=l(),oe=n("p"),Qt=s("Disable sliced VAE decoding. If "),Pe=n("code"),Ot=s("enable_vae_slicing"),qt=s(` was previously enabled, this method will go back to
computing decoding in one step.`),Ht=l(),U=n("div"),y(ne.$$.fragment),Kt=l(),ie=n("p"),eo=s("Disable tiled VAE decoding. If "),Ie=n("code"),to=s("enable_vae_tiling"),oo=s(` was previously enabled, this method will go back to
computing decoding in one step.`),no=l(),$=n("div"),y(ae.$$.fragment),io=l(),v=n("p"),ao=s(`Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
to `),Se=n("code"),so=s("enable_sequential_cpu_offload"),ro=s(", this method moves one whole model at a time to the GPU when its "),ke=n("code"),lo=s("forward"),po=s(`
method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
`),Xe=n("code"),co=s("enable_sequential_cpu_offload"),mo=s(", but performance is much better due to the iterative execution of the "),je=n("code"),fo=s("unet"),go=s("."),uo=l(),Z=n("div"),y(se.$$.fragment),ho=l(),Ee=n("p"),_o=s(`Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.`),bo=l(),W=n("div"),y(re.$$.fragment),vo=l(),Je=n("p"),yo=s(`Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
processing larger images.`),wo=l(),G=n("div"),y(le.$$.fragment),xo=l(),Ue=n("p"),Co=s("Encodes the prompt into text encoder hidden states."),this.h()},l(t){const m=dn('[data-svelte="svelte-1phssyn"]',document.head);h=i(m,"META",{name:!0,content:!0}),m.forEach(o),k=d(t),b=i(t,"H1",{class:!0});var de=a(b);_=i(de,"A",{id:!0,class:!0,href:!0});var No=a(_);N=i(No,"SPAN",{});var Mo=a(N);w(u.$$.fragment,Mo),Mo.forEach(o),No.forEach(o),M=d(de),ye=i(de,"SPAN",{});var Do=a(ye);pt=r(Do,"ControlNet with Stable Diffusion XL"),Do.forEach(o),de.forEach(o),Ze=d(t),F=i(t,"P",{});var To=a(F);z=i(To,"A",{href:!0,rel:!0});var Po=a(z);ct=r(Po,"Adding Conditional Control to Text-to-Image Diffusion Models"),Po.forEach(o),mt=r(To," by Lvmin Zhang and Maneesh Agrawala."),To.forEach(o),We=d(t),pe=i(t,"P",{});var Io=a(pe);ft=r(Io,"Using a pretrained model, we can provide control images (for example, a depth map) to control Stable Diffusion text-to-image generation so that it follows the structure of the depth image and fills in the details."),Io.forEach(o),Ge=d(t),ce=i(t,"P",{});var So=a(ce);gt=r(So,"The abstract from the paper is:"),So.forEach(o),Be=d(t),me=i(t,"P",{});var ko=a(me);we=i(ko,"EM",{});var Xo=a(we);ut=r(Xo,"We present a neural network structure, ControlNet, to control pretrained large diffusion models to support additional input conditions. The ControlNet learns task-specific conditions in an end-to-end way, and the learning is robust even when the training dataset is small (< 50k). Moreover, training a ControlNet is as fast as fine-tuning a diffusion model, and the model can be trained on a personal devices. Alternatively, if powerful computation clusters are available, the model can scale to large amounts (millions to billions) of data. We report that large diffusion models like Stable Diffusion can be augmented with ControlNets to enable conditional inputs like edge maps, segmentation maps, keypoints, etc. This may enrich the methods to control large diffusion models and further facilitate related applications."),Xo.forEach(o),ko.forEach(o),Fe=d(t),X=i(t,"P",{});var Oe=a(X);ht=r(Oe,"We provide support using ControlNets with "),fe=i(Oe,"A",{href:!0});var jo=a(fe);_t=r(jo,"Stable Diffusion XL"),jo.forEach(o),bt=r(Oe," (SDXL)."),Oe.forEach(o),ze=d(t),D=i(t,"P",{});var be=a(D);vt=r(be,"There are not many ControlNet checkpoints that are compatible with SDXL at the moment. So, we trained one using Canny edge maps as the conditioning images. To know more, check out the "),A=i(be,"A",{href:!0,rel:!0});var Eo=a(A);yt=r(Eo,"model card"),Eo.forEach(o),wt=r(be,". We encourage you to train custom ControlNets; we provide a "),V=i(be,"A",{href:!0,rel:!0});var Jo=a(V);xt=r(Jo,"training script"),Jo.forEach(o),Ct=r(be," for this."),be.forEach(o),Ae=d(t),ge=i(t,"P",{});var Uo=a(ge);Tt=r(Uo,"You can find some results below:"),Uo.forEach(o),Ve=d(t),R=i(t,"IMG",{src:!0,width:!0}),Re=d(t),S=i(t,"H2",{class:!0});var qe=a(S);j=i(qe,"A",{id:!0,class:!0,href:!0});var $o=a(j);xe=i($o,"SPAN",{});var Zo=a(xe);w(Y.$$.fragment,Zo),Zo.forEach(o),$o.forEach(o),Lt=d(qe),Ce=i(qe,"SPAN",{});var Wo=a(Ce);Nt=r(Wo,"StableDiffusionXLControlNetPipeline"),Wo.forEach(o),qe.forEach(o),Ye=d(t),c=i(t,"DIV",{class:!0});var f=a(c);w(Q.$$.fragment,f),Mt=d(f),Te=i(f,"P",{});var Go=a(Te);Dt=r(Go,"Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance."),Go.forEach(o),Pt=d(f),O=i(f,"P",{});var He=a(O);It=r(He,"This model inherits from "),ue=i(He,"A",{href:!0});var Bo=a(ue);St=r(Bo,"DiffusionPipeline"),Bo.forEach(o),kt=r(He,`. Check the superclass documentation for the generic methods the
library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)`),He.forEach(o),Xt=d(f),Le=i(f,"P",{});var Fo=a(Le);jt=r(Fo,"In addition the pipeline inherits the following loading methods:"),Fo.forEach(o),Et=d(f),q=i(f,"UL",{});var Ke=a(q);H=i(Ke,"LI",{});var et=a(H);Ne=i(et,"EM",{});var zo=a(Ne);Jt=r(zo,"Textual-Inversion"),zo.forEach(o),Ut=r(et,": "),he=i(et,"A",{href:!0});var Ao=a(he);$t=r(Ao,"loaders.TextualInversionLoaderMixin.load_textual_inversion()"),Ao.forEach(o),et.forEach(o),Zt=d(Ke),K=i(Ke,"LI",{});var tt=a(K);Me=i(tt,"EM",{});var Vo=a(Me);Wt=r(Vo,"LoRA"),Vo.forEach(o),Gt=r(tt,": "),_e=i(tt,"A",{href:!0});var Ro=a(_e);Bt=r(Ro,"loaders.LoraLoaderMixin.load_lora_weights()"),Ro.forEach(o),tt.forEach(o),Ke.forEach(o),Ft=d(f),P=i(f,"DIV",{class:!0});var ve=a(P);w(ee.$$.fragment,ve),zt=d(ve),De=i(ve,"P",{});var Yo=a(De);At=r(Yo,"Function invoked when calling the pipeline for generation."),Yo.forEach(o),Vt=d(ve),w(E.$$.fragment,ve),ve.forEach(o),Rt=d(f),J=i(f,"DIV",{class:!0});var ot=a(J);w(te.$$.fragment,ot),Yt=d(ot),oe=i(ot,"P",{});var nt=a(oe);Qt=r(nt,"Disable sliced VAE decoding. If "),Pe=i(nt,"CODE",{});var Qo=a(Pe);Ot=r(Qo,"enable_vae_slicing"),Qo.forEach(o),qt=r(nt,` was previously enabled, this method will go back to
computing decoding in one step.`),nt.forEach(o),ot.forEach(o),Ht=d(f),U=i(f,"DIV",{class:!0});var it=a(U);w(ne.$$.fragment,it),Kt=d(it),ie=i(it,"P",{});var at=a(ie);eo=r(at,"Disable tiled VAE decoding. If "),Ie=i(at,"CODE",{});var Oo=a(Ie);to=r(Oo,"enable_vae_tiling"),Oo.forEach(o),oo=r(at,` was previously enabled, this method will go back to
computing decoding in one step.`),at.forEach(o),it.forEach(o),no=d(f),$=i(f,"DIV",{class:!0});var st=a($);w(ae.$$.fragment,st),io=d(st),v=i(st,"P",{});var I=a(v);ao=r(I,`Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
to `),Se=i(I,"CODE",{});var qo=a(Se);so=r(qo,"enable_sequential_cpu_offload"),qo.forEach(o),ro=r(I,", this method moves one whole model at a time to the GPU when its "),ke=i(I,"CODE",{});var Ho=a(ke);lo=r(Ho,"forward"),Ho.forEach(o),po=r(I,`
method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
`),Xe=i(I,"CODE",{});var Ko=a(Xe);co=r(Ko,"enable_sequential_cpu_offload"),Ko.forEach(o),mo=r(I,", but performance is much better due to the iterative execution of the "),je=i(I,"CODE",{});var en=a(je);fo=r(en,"unet"),en.forEach(o),go=r(I,"."),I.forEach(o),st.forEach(o),uo=d(f),Z=i(f,"DIV",{class:!0});var rt=a(Z);w(se.$$.fragment,rt),ho=d(rt),Ee=i(rt,"P",{});var tn=a(Ee);_o=r(tn,`Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.`),tn.forEach(o),rt.forEach(o),bo=d(f),W=i(f,"DIV",{class:!0});var lt=a(W);w(re.$$.fragment,lt),vo=d(lt),Je=i(lt,"P",{});var on=a(Je);yo=r(on,`Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
processing larger images.`),on.forEach(o),lt.forEach(o),wo=d(f),G=i(f,"DIV",{class:!0});var dt=a(G);w(le.$$.fragment,dt),xo=d(dt),Ue=i(dt,"P",{});var nn=a(Ue);Co=r(nn,"Encodes the prompt into text encoder hidden states."),nn.forEach(o),dt.forEach(o),f.forEach(o),this.h()},h(){p(h,"name","hf:doc:metadata"),p(h,"content",JSON.stringify(_n)),p(_,"id","controlnet-with-stable-diffusion-xl"),p(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(_,"href","#controlnet-with-stable-diffusion-xl"),p(b,"class","relative group"),p(z,"href","https://huggingface.co/papers/2302.05543"),p(z,"rel","nofollow"),p(fe,"href","./stable_diffusion/stable_diffusion_xl.md"),p(A,"href","https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0"),p(A,"rel","nofollow"),p(V,"href","https://github.com/huggingface/diffusers/blob/main/examples/controlnet/README_sdxl.md"),p(V,"rel","nofollow"),pn(R.src,Lo="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/sd_xl/sdxl_controlnet_canny_grid.png")||p(R,"src",Lo),p(R,"width","600"),p(j,"id","diffusers.StableDiffusionXLControlNetPipeline"),p(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(j,"href","#diffusers.StableDiffusionXLControlNetPipeline"),p(S,"class","relative group"),p(ue,"href","/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline"),p(he,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/depth2img#diffusers.StableDiffusionDepth2ImgPipeline.load_textual_inversion"),p(_e,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/depth2img#diffusers.StableDiffusionDepth2ImgPipeline.load_lora_weights"),p(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(c,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,h),g(t,k,m),g(t,b,m),e(b,_),e(_,N),x(u,N,null),e(b,M),e(b,ye),e(ye,pt),g(t,Ze,m),g(t,F,m),e(F,z),e(z,ct),e(F,mt),g(t,We,m),g(t,pe,m),e(pe,ft),g(t,Ge,m),g(t,ce,m),e(ce,gt),g(t,Be,m),g(t,me,m),e(me,we),e(we,ut),g(t,Fe,m),g(t,X,m),e(X,ht),e(X,fe),e(fe,_t),e(X,bt),g(t,ze,m),g(t,D,m),e(D,vt),e(D,A),e(A,yt),e(D,wt),e(D,V),e(V,xt),e(D,Ct),g(t,Ae,m),g(t,ge,m),e(ge,Tt),g(t,Ve,m),g(t,R,m),g(t,Re,m),g(t,S,m),e(S,j),e(j,xe),x(Y,xe,null),e(S,Lt),e(S,Ce),e(Ce,Nt),g(t,Ye,m),g(t,c,m),x(Q,c,null),e(c,Mt),e(c,Te),e(Te,Dt),e(c,Pt),e(c,O),e(O,It),e(O,ue),e(ue,St),e(O,kt),e(c,Xt),e(c,Le),e(Le,jt),e(c,Et),e(c,q),e(q,H),e(H,Ne),e(Ne,Jt),e(H,Ut),e(H,he),e(he,$t),e(q,Zt),e(q,K),e(K,Me),e(Me,Wt),e(K,Gt),e(K,_e),e(_e,Bt),e(c,Ft),e(c,P),x(ee,P,null),e(P,zt),e(P,De),e(De,At),e(P,Vt),x(E,P,null),e(c,Rt),e(c,J),x(te,J,null),e(J,Yt),e(J,oe),e(oe,Qt),e(oe,Pe),e(Pe,Ot),e(oe,qt),e(c,Ht),e(c,U),x(ne,U,null),e(U,Kt),e(U,ie),e(ie,eo),e(ie,Ie),e(Ie,to),e(ie,oo),e(c,no),e(c,$),x(ae,$,null),e($,io),e($,v),e(v,ao),e(v,Se),e(Se,so),e(v,ro),e(v,ke),e(ke,lo),e(v,po),e(v,Xe),e(Xe,co),e(v,mo),e(v,je),e(je,fo),e(v,go),e(c,uo),e(c,Z),x(se,Z,null),e(Z,ho),e(Z,Ee),e(Ee,_o),e(c,bo),e(c,W),x(re,W,null),e(W,vo),e(W,Je),e(Je,yo),e(c,wo),e(c,G),x(le,G,null),e(G,xo),e(G,Ue),e(Ue,Co),Qe=!0},p(t,[m]){const de={};m&2&&(de.$$scope={dirty:m,ctx:t}),E.$set(de)},i(t){Qe||(C(u.$$.fragment,t),C(Y.$$.fragment,t),C(Q.$$.fragment,t),C(ee.$$.fragment,t),C(E.$$.fragment,t),C(te.$$.fragment,t),C(ne.$$.fragment,t),C(ae.$$.fragment,t),C(se.$$.fragment,t),C(re.$$.fragment,t),C(le.$$.fragment,t),Qe=!0)},o(t){T(u.$$.fragment,t),T(Y.$$.fragment,t),T(Q.$$.fragment,t),T(ee.$$.fragment,t),T(E.$$.fragment,t),T(te.$$.fragment,t),T(ne.$$.fragment,t),T(ae.$$.fragment,t),T(se.$$.fragment,t),T(re.$$.fragment,t),T(le.$$.fragment,t),Qe=!1},d(t){o(h),t&&o(k),t&&o(b),L(u),t&&o(Ze),t&&o(F),t&&o(We),t&&o(pe),t&&o(Ge),t&&o(ce),t&&o(Be),t&&o(me),t&&o(Fe),t&&o(X),t&&o(ze),t&&o(D),t&&o(Ae),t&&o(ge),t&&o(Ve),t&&o(R),t&&o(Re),t&&o(S),L(Y),t&&o(Ye),t&&o(c),L(Q),L(ee),L(E),L(te),L(ne),L(ae),L(se),L(re),L(le)}}}const _n={local:"controlnet-with-stable-diffusion-xl",sections:[{local:"diffusers.StableDiffusionXLControlNetPipeline",title:"StableDiffusionXLControlNetPipeline"}],title:"ControlNet with Stable Diffusion XL"};function bn($e){return cn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Tn extends sn{constructor(h){super();rn(this,h,bn,hn,ln,{})}}export{Tn as default,_n as metadata};
