KenSchae
/
Pallaidium
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Add: Playground v2

pull/97/head
tin2tin 12 months ago committed by GitHub
parent
e0f0d032c0
commit
075d695965
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 109 additions and 9 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 118
      __init__.py

118
__init__.py
Unescape View File

@ -729,6 +729,7 @@ def install_modules(self):
import_module(self, "PIL", "pillow")
import_module(self, "scipy", "scipy")
import_module(self, "IPython", "IPython")
import_module(self, "omegaconf", "omegaconf")
# import_module(self, "mustango", "mustango")
# import_module(self, "mustango", "git+https://github.com/AMAAI-Lab/mustango.git")
# subprocess.check_call([pybin, "-m", "pip", "install", "mediapipe", "--upgrade"])
@ -1062,6 +1063,11 @@ class GeneratorAddonPreferences(AddonPreferences):
# "stabilityai/stable-diffusion-xl-base-1.0",
# "Img2img SD XL 1.0 Refine (1024x1024)",
# "Stable Diffusion XL 1.0",
# ),
# (
# "stabilityai/sd-turbo",
# "Img2img SD Turbo (512x512)",
# "stabilityai/sd-turbo",
# ),
#("camenduru/potat1", "Potat v1 (1024x576)", "Potat (1024x576)"),
# ("VideoCrafter/Image2Video-512", "VideoCrafter v1 (512x512)", "VideoCrafter/Image2Video-512"),
@ -1133,6 +1139,7 @@ class GeneratorAddonPreferences(AddonPreferences):
),
# ("ptx0/terminus-xl-gamma-v1", "Terminus XL Gamma v1", "ptx0/terminus-xl-gamma-v1"),
("warp-ai/wuerstchen", "Würstchen (1024x1024)", "warp-ai/wuerstchen"),
("playgroundai/playground-v2-1024px-aesthetic", "Playground v2 (1024x1024)", "playgroundai/playground-v2-1024px-aesthetic"),
(
"Salesforce/blipdiffusion",
"Blip Subject Driven (512x512)",
@ -1815,12 +1822,6 @@ class SEQUENCER_PT_pallaidium_panel(Panel): # UI
and movie_model_card
== "stabilityai/stable-video-diffusion-img2vid-xt"
)
or (
type == "image" and image_model_card == "stabilityai/sdxl-turbo"
)
or (
type == "image" and image_model_card == "stabilityai/sd-turbo"
)
):
pass
else:
@ -1915,6 +1916,11 @@ class SEQUENCER_PT_pallaidium_panel(Panel): # UI
# "sequencer.text_to_generator", text="Generate from Strips"
# )
# else:
if movie_model_card == "stabilityai/sd-turbo":
row.operator(
"sequencer.text_to_generator", text="Generate from Strips"
)
else:
row.operator("sequencer.generate_movie", text="Generate")
if type == "image":
row.operator("sequencer.generate_image", text="Generate")
@ -2005,6 +2011,65 @@ class SEQUENCER_OT_generate_movie(Operator):
and input == "input_strips"
and movie_model_card != "guoyww/animatediff-motion-adapter-v1-5-2"
):
if (
movie_model_card == "stabilityai/sd-turbo"
): # img2img
# from diffusers import StableDiffusionXLImg2ImgPipeline, AutoencoderKL
# vae = AutoencoderKL.from_pretrained(
# "madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16
# )
from diffusers import AutoPipelineForImage2Image
#from diffusers.utils import load_image
pipe = AutoPipelineForImage2Image.from_pretrained("stabilityai/sd-turbo", torch_dtype=torch.float16, variant="fp16")
#pipe.to("cuda")
#init_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png").resize((512, 512))
#prompt = "cat wizard, gandalf, lord of the rings, detailed, fantasy, cute, adorable, Pixar, Disney, 8k"
#image = pipe(prompt, image=init_image, num_inference_steps=2, strength=0.5, guidance_scale=0.0).images[0]
# pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(
# movie_model_card,
# torch_dtype=torch.float16,
# variant="fp16",
# vae=vae,
# )
# from diffusers import DPMSolverMultistepScheduler
# pipe.scheduler = DPMSolverMultistepScheduler.from_config(
# pipe.scheduler.config
# )
# pipe.watermark = NoWatermark()
if low_vram():
pipe.enable_model_cpu_offload()
# pipe.unet.enable_forward_chunking(chunk_size=1, dim=1) # Heavy
# pipe.enable_vae_slicing()
else:
pipe.to(gfx_device)
# from diffusers import StableDiffusionXLImg2ImgPipeline
# refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained(
# "stabilityai/stable-diffusion-xl-refiner-1.0",
# text_encoder_2=pipe.text_encoder_2,
# vae=pipe.vae,
# torch_dtype=torch.float16,
# variant="fp16",
# )
# if low_vram():
# refiner.enable_model_cpu_offload()
# # refiner.enable_vae_tiling()
# # refiner.enable_vae_slicing()
# else:
# refiner.to(gfx_device)
# if (
# movie_model_card == "stabilityai/stable-diffusion-xl-base-1.0"
# ): # img2img
@ -2050,7 +2115,7 @@ class SEQUENCER_OT_generate_movie(Operator):
# # refiner.enable_vae_slicing()
# else:
# refiner.to(gfx_device)
if (
elif (
movie_model_card == "stabilityai/stable-video-diffusion-img2vid"
or movie_model_card == "stabilityai/stable-video-diffusion-img2vid-xt"
):
@ -2290,9 +2355,44 @@ class SEQUENCER_OT_generate_movie(Operator):
# torch.cuda.empty_cache()
# video_frames = np.array(video_frames)
# img2img
if movie_model_card == "stabilityai/sd-turbo":
print("Process: Frame by frame (SD Turbo)")
input_video_path = video_path
output_video_path = solve_path("temp_images")
if scene.movie_path:
frames = process_video(input_video_path, output_video_path)
elif scene.image_path:
frames = process_image(
scene.image_path, int(scene.generate_movie_frames)
)
video_frames = []
# Iterate through the frames
for frame_idx, frame in enumerate(frames): # would love to get this flicker free
print(str(frame_idx + 1) + "/" + str(len(frames)))
image = pipe(
prompt,
negative_prompt=negative_prompt,
num_inference_steps=2, #movie_num_inference_steps,
strength=0.5,#scene.image_power,
guidance_scale=3.0,
image=frame,
generator=generator,
).images[0]
video_frames.append(image)
if torch.cuda.is_available():
torch.cuda.empty_cache()
video_frames = np.array(video_frames)
# vid2vid / img2vid
if (
elif (
movie_model_card == "stabilityai/stable-video-diffusion-img2vid"
or movie_model_card
== "stabilityai/stable-video-diffusion-img2vid-xt"
@ -4007,7 +4107,7 @@ class SEQUENCER_OT_strip_to_generatorAI(Operator):
self.report({"INFO"}, "Select strip(s) for processing.")
return {"CANCELLED"}
else:
print("\nStrip input processing started (ctrl+c to cancel).")
print("\nStrip input processing started (Ctrl+C to Cancel).")
for strip in strips:
if strip.type in {"MOVIE", "IMAGE", "TEXT", "SCENE"}:
break

Write Preview
Loading…
Cancel
Save