[ip-adapter] fix ip-adapter for StableDiffusionInstructPix2PixPipeline

[yiyixuxu]

fix https://github.com/huggingface/diffusers/issues/7799 this works now

import requests
from io import BytesIO
from PIL import Image
import torch

from diffusers import StableDiffusionInstructPix2PixPipeline

pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained(
    "timbrooks/instruct-pix2pix", torch_dtype=torch.float16, variant="fp16"
)
pipe = pipe.to("cuda")
pipe.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin")
pipe.set_ip_adapter_scale(0.6)

def download_image(url):
    response = requests.get(url)
    return Image.open(BytesIO(response.content)).convert("RGB")

img_url = "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png"
image = download_image(img_url).resize((512, 512))

url = "https://images.nightcafe.studio/jobs/j2VTkZXvVFoOciX20IqZ/j2VTkZXvVFoOciX20IqZ--1--opa80.jpg"
ip_adapter_image = download_image(url).resize((512, 512))

prompt = "make the mountains snowy"
image = pipe(prompt=prompt, image=image, ip_adapter_image=ip_adapter_image, num_inference_steps=100, guidance_scale=6, image_guidance_scale=1).images[0]
image.save(f"yiyi_test_1_out.png")

[HuggingFaceDocBuilderDev]

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.