bl_info = {
" name " : " Pallaidium - Generative AI " ,
" author " : " tintwotin " ,
" version " : ( 1 , 8 ) ,
" blender " : ( 3 , 4 , 0 ) ,
" location " : " Video Sequence Editor > Sidebar > Generative AI " ,
" description " : " AI Generate media in the VSE " ,
" category " : " Sequencer " ,
}
# TO DO: Style title check, long prompts, SDXL controlnet,Move prints.
import bpy , ctypes , random
from bpy . types import Operator , Panel , AddonPreferences , UIList , PropertyGroup
from bpy . props import (
StringProperty ,
BoolProperty ,
EnumProperty ,
IntProperty ,
FloatProperty ,
)
import site , platform , json
import subprocess
import sys , os , aud , re
import string
from os . path import dirname , realpath , isdir , join , basename
import shutil
from datetime import date
try :
exec ( " import torch " )
if torch . cuda . is_available ( ) :
gfx_device = " cuda "
elif torch . backends . mps . is_available ( ) :
gfx_device = " mps "
else :
gfx_device = " cpu "
except ModuleNotFoundError :
print ( " Pallaidium dependencies needs to be installed and then Blender needs to be restarted. " )
os_platform = platform . system ( ) # 'Linux', 'Darwin', 'Java', 'Windows'
def show_system_console ( show ) :
if os_platform == " Windows " :
# https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-showwindow
SW_HIDE = 0
SW_SHOW = 5
ctypes . windll . user32 . ShowWindow (
ctypes . windll . kernel32 . GetConsoleWindow ( ) , SW_SHOW # if show else SW_HIDE
)
def set_system_console_topmost ( top ) :
if os_platform == " Windows " :
# https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-setwindowpos
HWND_NOTOPMOST = - 2
HWND_TOPMOST = - 1
HWND_TOP = 0
SWP_NOMOVE = 0x0002
SWP_NOSIZE = 0x0001
SWP_NOZORDER = 0x0004
ctypes . windll . user32 . SetWindowPos (
ctypes . windll . kernel32 . GetConsoleWindow ( ) ,
HWND_TOP if top else HWND_NOTOPMOST ,
0 ,
0 ,
0 ,
0 ,
SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER ,
)
# normalize text, remove redundant whitespace and convert non-ascii quotes to ascii
def split_and_recombine_text ( text , desired_length = 200 , max_length = 300 ) :
""" Split text it into chunks of a desired length trying to keep sentences intact. """
text = re . sub ( r " \ n \ n+ " , " \n " , text )
text = re . sub ( r " \ s+ " , " " , text )
text = re . sub ( r " [“”] " , ' " ' , text )
rv = [ ]
in_quote = False
current = " "
split_pos = [ ]
pos = - 1
end_pos = len ( text ) - 1
def seek ( delta ) :
nonlocal pos , in_quote , current
is_neg = delta < 0
for _ in range ( abs ( delta ) ) :
if is_neg :
pos - = 1
current = current [ : - 1 ]
else :
pos + = 1
current + = text [ pos ]
if text [ pos ] == ' " ' :
in_quote = not in_quote
return text [ pos ]
def peek ( delta ) :
p = pos + delta
return text [ p ] if p < end_pos and p > = 0 else " "
def commit ( ) :
nonlocal rv , current , split_pos
rv . append ( current )
current = " "
split_pos = [ ]
while pos < end_pos :
c = seek ( 1 )
# do we need to force a split?
if len ( current ) > = max_length :
if len ( split_pos ) > 0 and len ( current ) > ( desired_length / 2 ) :
# we have at least one sentence and we are over half the desired length, seek back to the last split
d = pos - split_pos [ - 1 ]
seek ( - d )
else :
# no full sentences, seek back until we are not in the middle of a word and split there
while c not in " !?., \n " and pos > 0 and len ( current ) > desired_length :
c = seek ( - 1 )
commit ( )
# check for sentence boundaries
elif not in_quote and ( c in " !? \n " or ( c == " . " and peek ( 1 ) in " \n " ) ) :
# seek forward if we have consecutive boundary markers but still within the max length
while (
pos < len ( text ) - 1 and len ( current ) < max_length and peek ( 1 ) in " !?., "
) :
c = seek ( 1 )
split_pos . append ( pos )
if len ( current ) > = desired_length :
commit ( )
# treat end of quote as a boundary if its followed by a space or newline
elif in_quote and peek ( 1 ) == ' " ' and peek ( 2 ) in " \n " :
seek ( 2 )
split_pos . append ( pos )
rv . append ( current )
# clean up, remove lines with only whitespace or punctuation
rv = [ s . strip ( ) for s in rv ]
rv = [ s for s in rv if len ( s ) > 0 and not re . match ( r " ^[ \ s \ .,;:!?]*$ " , s ) ]
return rv
def extract_numbers ( input_string ) :
numbers = re . findall ( r " \ d+ " , input_string )
if numbers :
return int ( numbers [ 0 ] )
else :
return None
def load_styles ( json_filename ) :
styles_array = [ ]
try :
with open ( json_filename , " r " ) as json_file :
data = json . load ( json_file )
except FileNotFoundError :
print ( f " JSON file ' { json_filename } ' not found. " )
data = [ ]
for item in data :
name = item [ " name " ]
prompt = item [ " prompt " ]
negative_prompt = item [ " negative_prompt " ]
styles_array . append (
( negative_prompt . lower ( ) . replace ( " " , " _ " ) , name . title ( ) , prompt )
)
return styles_array
def style_prompt ( prompt ) :
selected_entry_key = bpy . context . scene . generatorai_styles
return_array = [ ]
if selected_entry_key :
styles_array = load_styles (
os . path . dirname ( os . path . abspath ( __file__ ) ) + " /styles.json "
)
if styles_array :
selected_entry = next (
( item for item in styles_array if item [ 0 ] == selected_entry_key ) , None
)
if selected_entry :
selected_entry_list = list ( selected_entry )
return_array . append ( selected_entry_list [ 2 ] . replace ( " {prompt} " , prompt ) )
return_array . append ( selected_entry_list [ 0 ] . replace ( " _ " , " " ) )
return return_array
return_array . append ( prompt )
return_array . append ( bpy . context . scene . generate_movie_negative_prompt )
return return_array
def closest_divisible_32 ( num ) :
# Determine the remainder when num is divided by 64
remainder = num % 32
# If the remainder is less than or equal to 16, return num - remainder,
# but ensure the result is not less than 192
if remainder < = 16 :
result = num - remainder
return max ( result , 192 )
# Otherwise, return num + (32 - remainder)
else :
return max ( num + ( 32 - remainder ) , 192 )
def closest_divisible_128 ( num ) :
# Determine the remainder when num is divided by 128
remainder = num % 128
# If the remainder is less than or equal to 64, return num - remainder,
# but ensure the result is not less than 256
if remainder < = 64 :
result = num - remainder
return max ( result , 256 )
# Otherwise, return num + (32 - remainder)
else :
return max ( num + ( 64 - remainder ) , 256 )
def find_first_empty_channel ( start_frame , end_frame ) :
for ch in range ( 1 , len ( bpy . context . scene . sequence_editor . sequences_all ) + 1 ) :
for seq in bpy . context . scene . sequence_editor . sequences_all :
if (
seq . channel == ch
and seq . frame_final_start < end_frame
and ( seq . frame_final_start + seq . frame_final_duration ) > start_frame
) :
break
else :
return ch
return 1
def clean_filename ( filename ) :
filename = filename [ : 50 ]
valid_chars = " -_,.() %s %s " % ( string . ascii_letters , string . digits )
clean_filename = " " . join ( c if c in valid_chars else " _ " for c in filename )
clean_filename = clean_filename . replace ( " \n " , " " )
clean_filename = clean_filename . replace ( " \r " , " " )
clean_filename = clean_filename . replace ( " " , " _ " )
return clean_filename . strip ( )
def create_folder ( folderpath ) :
try :
os . makedirs ( folderpath )
return True
except FileExistsError :
# directory already exists
pass
return False
def solve_path ( full_path ) :
preferences = bpy . context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
name , ext = os . path . splitext ( full_path )
dir_path , filename = os . path . split ( name )
dir_path = os . path . join ( addon_prefs . generator_ai , str ( date . today ( ) ) )
create_folder ( dir_path )
cleaned_filename = clean_filename ( filename )
new_filename = cleaned_filename + ext
i = 1
while os . path . exists ( os . path . join ( dir_path , new_filename ) ) :
name , ext = os . path . splitext ( new_filename )
new_filename = f " { name . rsplit ( ' ( ' , 1 ) [ 0 ] } ( { i } ) { ext } "
i + = 1
return os . path . join ( dir_path , new_filename )
def limit_string ( my_string ) :
if len ( my_string ) > 77 :
print (
" Warning: String is longer than 77 characters. Excessive string: " ,
my_string [ 77 : ] ,
)
return my_string [ : 77 ]
else :
return my_string
def delete_strip ( input_strip ) :
if input_strip is None :
return
original_selection = [
strip
for strip in bpy . context . scene . sequence_editor . sequences_all
if strip . select
]
bpy . ops . sequencer . select_all ( action = " DESELECT " )
input_strip . select = True
bpy . ops . sequencer . delete ( )
for strip in original_selection :
strip . select = True
def load_video_as_np_array ( video_path ) :
import cv2
import numpy as np
cap = cv2 . VideoCapture ( video_path )
if not cap . isOpened ( ) :
raise IOError ( " Error opening video file " )
frames = [ ]
while True :
ret , frame = cap . read ( )
if not ret :
break
frame = cv2 . cvtColor ( frame , cv2 . COLOR_BGR2RGB )
frames . append ( frame )
cap . release ( )
return np . array ( frames )
def load_first_frame ( file_path ) :
import cv2 , PIL , os
from diffusers . utils import load_image
extension = os . path . splitext ( file_path ) [
- 1
] . lower ( ) # Convert to lowercase for case-insensitive comparison
valid_image_extensions = {
" .sgi " ,
" .rgb " ,
" .bw " ,
" .cin " ,
" .dpx " ,
" .png " ,
" .jpg " ,
" .jpeg " ,
" .jp2 " ,
" .jp2 " ,
" .j2c " ,
" .tga " ,
" .exr " ,
" .hdr " ,
" .tif " ,
" .tiff " ,
" .webp " ,
}
valid_video_extensions = {
" .avi " ,
" .flc " ,
" .mov " ,
" .movie " ,
" .mp4 " ,
" .m4v " ,
" .m2v " ,
" .m2t " ,
" .m2ts " ,
" .mts " ,
" .ts " ,
" .mv " ,
" .avs " ,
" .wmv " ,
" .ogv " ,
" .ogg " ,
" .r3d " ,
" .dv " ,
" .mpeg " ,
" .mpg " ,
" .mpg2 " ,
" .vob " ,
" .mkv " ,
" .flv " ,
" .divx " ,
" .xvid " ,
" .mxf " ,
" .webm " ,
}
if extension in valid_image_extensions :
image = cv2 . imread ( file_path )
# if image is not None:
image = cv2 . cvtColor ( image , cv2 . COLOR_BGR2RGB )
return PIL . Image . fromarray ( image )
if extension in valid_video_extensions :
# Try to open the file as a video
cap = cv2 . VideoCapture ( file_path )
# Check if the file was successfully opened as a video
if cap . isOpened ( ) :
# Read the first frame from the video
ret , frame = cap . read ( )
cap . release ( ) # Release the video capture object
if ret :
# If the first frame was successfully read, it's a video
frame = cv2 . cvtColor ( frame , cv2 . COLOR_BGR2RGB )
return PIL . Image . fromarray ( frame )
# If neither video nor image worked, return None
return None
def process_frames ( frame_folder_path , target_width ) :
from PIL import Image
Image . MAX_IMAGE_PIXELS = None
import cv2
processed_frames = [ ]
# List all image files in the folder
image_files = sorted (
[ f for f in os . listdir ( frame_folder_path ) if f . endswith ( " .png " ) ]
)
for image_file in image_files :
image_path = os . path . join ( frame_folder_path , image_file )
img = Image . open ( image_path )
# Process the image (resize and convert to RGB)
frame_width , frame_height = img . size
# Calculate the target height to maintain the original aspect ratio
target_height = int ( ( target_width / frame_width ) * frame_height )
# Ensure width and height are divisible by 64
target_width = closest_divisible_32 ( target_width )
target_height = closest_divisible_32 ( target_height )
img = img . resize ( ( target_width , target_height ) , Image . Resampling . LANCZOS )
img = img . convert ( " RGB " )
processed_frames . append ( img )
return processed_frames
def process_video ( input_video_path , output_video_path ) :
from PIL import Image
Image . MAX_IMAGE_PIXELS = None
import cv2
import shutil
scene = bpy . context . scene
movie_x = scene . generate_movie_x
# Create a temporary folder for storing frames
temp_image_folder = solve_path ( " temp_images " )
if not os . path . exists ( temp_image_folder ) :
os . makedirs ( temp_image_folder )
# Open the video file using OpenCV
cap = cv2 . VideoCapture ( input_video_path )
frame_count = int ( cap . get ( cv2 . CAP_PROP_FRAME_COUNT ) )
fps = int ( cap . get ( cv2 . CAP_PROP_FPS ) )
# Save each loaded frame as an image in the temp folder
for i in range ( frame_count ) :
ret , frame = cap . read ( )
if not ret :
break
# Save the frame as an image in the temp folder
temp_image_path = os . path . join ( temp_image_folder , f " frame_ { i : 04d } .png " )
cv2 . imwrite ( temp_image_path , frame )
cap . release ( )
# Process frames using the separate function
processed_frames = process_frames ( temp_image_folder , movie_x )
# Clean up: Delete the temporary image folder
shutil . rmtree ( temp_image_folder )
return processed_frames
# Define the function for zooming effect
def zoomPan ( img , zoom = 1 , angle = 0 , coord = None ) :
import cv2
cy , cx = [ i / 2 for i in img . shape [ : - 1 ] ] if coord is None else coord [ : : - 1 ]
rot = cv2 . getRotationMatrix2D ( ( cx , cy ) , angle , zoom )
res = cv2 . warpAffine ( img , rot , img . shape [ 1 : : - 1 ] , flags = cv2 . INTER_LINEAR )
return res
def process_image ( image_path , frames_nr ) :
from PIL import Image
Image . MAX_IMAGE_PIXELS = None
import cv2 , shutil
scene = bpy . context . scene
movie_x = scene . generate_movie_x
img = cv2 . imread ( image_path )
height , width , layers = img . shape
# Create a temporary folder for storing frames
temp_image_folder = solve_path ( " /temp_images " )
if not os . path . exists ( temp_image_folder ) :
os . makedirs ( temp_image_folder )
max_zoom = 2.0 # Maximum Zoom level (should be > 1.0)
max_rot = 30 # Maximum rotation in degrees, set '0' for no rotation
# Make the loop for Zooming-in
i = 1
while i < frames_nr :
zLvl = 1.0 + ( ( i / ( 1 / ( max_zoom - 1 ) ) / frames_nr ) * 0.005 )
angle = 0 # i * max_rot / frames_nr
zoomedImg = zoomPan ( img , zLvl , angle , coord = None )
output_path = os . path . join ( temp_image_folder , f " frame_ { i : 04d } .png " )
cv2 . imwrite ( output_path , zoomedImg )
i = i + 1
# Process frames using the separate function
processed_frames = process_frames ( temp_image_folder , movie_x )
# Clean up: Delete the temporary image folder
shutil . rmtree ( temp_image_folder )
cv2 . destroyAllWindows ( )
return processed_frames
def low_vram ( ) :
import torch
total_vram = 0
for i in range ( torch . cuda . device_count ( ) ) :
properties = torch . cuda . get_device_properties ( i )
total_vram + = properties . total_memory
return ( total_vram / ( 1024 * * 3 ) ) < 6.1 # Y/N under 6.1 GB?
def isWindows ( ) :
return os . name == " nt "
def isMacOS ( ) :
return os . name == " posix " and platform . system ( ) == " Darwin "
def isLinux ( ) :
return os . name == " posix " and platform . system ( ) == " Linux "
def python_exec ( ) :
import sys
if isWindows ( ) :
return os . path . join ( sys . prefix , " bin " , " python.exe " )
elif isMacOS ( ) :
try :
# 2.92 and older
path = bpy . app . binary_path_python
except AttributeError :
# 2.93 and later
import sys
path = sys . executable
return os . path . abspath ( path )
elif isLinux ( ) :
return os . path . join ( sys . prefix , " sys.prefix/bin " , " python " )
else :
print ( " sorry, still not implemented for " , os . name , " - " , platform . system )
def import_module ( self , module , install_module ) :
show_system_console ( True )
set_system_console_topmost ( True )
module = str ( module )
python_exe = python_exec ( )
try :
subprocess . call ( [ python_exe , " import " , packageName ] )
# exec("import " + module)
except : # ModuleNotFoundError:
# app_path = site.USER_SITE
# if app_path not in sys.path:
# sys.path.append(app_path)
# pybin = sys.executable
# target = os.path.join(sys.prefix, 'site-packages')
# if target not in sys.path:
# sys.path.append(target)
self . report ( { " INFO " } , " Installing: " + module + " module. " )
print ( " Installing: " + module + " module " )
subprocess . call ( [ python_exe , " -m " , " pip " , " install " , install_module ] )
# subprocess.check_call(
# [
# pybin,
# "-m",
# "pip",
# "install",
# install_module,
# "--no-warn-script-location",
# "--user",
# #'-t', target,
# ]
# )
try :
exec ( " import " + module )
except ModuleNotFoundError :
return False
return True
def install_modules ( self ) :
os_platform = platform . system ( )
# app_path = site.USER_SITE
# if app_path not in sys.path:
# sys.path.append(app_path)
# pybin = sys.executable
# target = os.path.join(sys.prefix, 'site-packages')
# if target not in sys.path:
# sys.path.append(target)
pybin = python_exec ( )
print ( " Ensuring: pip " )
try :
subprocess . call ( [ pybin , " -m " , " ensurepip " ] )
subprocess . call ( [ pybin , " -m " , " pip " , " install " , " --upgrade " , " pip " ] )
except ImportError :
pass
#!pip install lmdb
#!pip install torch==2.1.0+cu121 torchvision==0.16.0+cu121 torchaudio==2.1.0 torchtext==0.16.0+cpu torchdata==0.7.0 --index-url https://download.pytorch.org/whl/cu121
# import_module(self, "xformers", "xformers")
try :
exec ( " import torch " )
except ModuleNotFoundError :
subprocess . call ( [ pybin , " -m " , " pip " , " install " , " lmdb " ] )
self . report ( { " INFO " } , " Installing: torch module. " )
print ( " Installing: torch module " )
if os_platform == " Windows " :
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" torch==2.1.0+cu121 " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu121 " ,
" --no-warn-script-location " ,
# "--user",
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" torchvision==0.16.0+cu121 " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu121 " ,
" --no-warn-script-location " ,
# "--user",
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" torchaudio==2.1.0 " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu121 " ,
" --no-warn-script-location " ,
# "--user",
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" -U " ,
" xformers " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu121 " ,
" --no-warn-script-location " ,
# "--user",
]
)
else :
import_module ( self , " torch " , " torch " )
import_module ( self , " torchvision " , " torchvision " )
import_module ( self , " torchaudio " , " torchaudio " )
import_module ( self , " huggingface_hub " , " huggingface_hub " )
import_module (
self , " accelerate " , " git+https://github.com/huggingface/accelerate.git "
)
# import_module(self, "transformers", "git+https://github.com/huggingface/transformers.git")
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" git+https://github.com/huggingface/transformers.git " ,
" --upgrade " ,
]
)
# subprocess.check_call([pybin, "-m", "pip", "install", "transformers", "--upgrade"])
import_module ( self , " bark " , " git+https://github.com/suno-ai/bark.git " )
# import_module(self, "bark", "git+https://github.com/suno-ai/bark.git")
import_module ( self , " diffusers " , " diffusers " )
# import_module(self, "diffusers", "git+https://github.com/huggingface/diffusers.git@v0.22.3")
import_module ( self , " tensorflow " , " tensorflow " )
# if os_platform == "Darwin" or os_platform == "Linux":
# import_module(self, "sox", "sox")
# else:
import_module ( self , " soundfile " , " PySoundFile " )
# import_module(self, "transformers", "transformers")
import_module ( self , " sentencepiece " , " sentencepiece " )
import_module ( self , " safetensors " , " safetensors " )
import_module ( self , " cv2 " , " opencv_python " )
import_module ( self , " PIL " , " pillow " )
import_module ( self , " scipy " , " scipy " )
import_module ( self , " IPython " , " IPython " )
import_module ( self , " omegaconf " , " omegaconf " )
# import_module(self, "mustango", "mustango")
# import_module(self, "mustango", "git+https://github.com/AMAAI-Lab/mustango.git")
# subprocess.check_call([pybin, "-m", "pip", "install", "mediapipe", "--upgrade"])
# try:
# import_module(self, "mediapipe", "git+https://github.com/google/mediapipe.git")
# except ImportError:
# pass
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
# "bark",
# "--index-url",
" git+https://github.com/suno-ai/bark.git " ,
" --no-warn-script-location " ,
" --user " ,
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" xformers==0.0.22.post4 " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu118 " ,
" --no-warn-script-location " ,
" --user " ,
]
)
import_module ( self , " imageio " , " imageio " )
import_module ( self , " imwatermark " , " invisible-watermark>=0.2.0 " )
if os_platform == " Windows " :
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" libtorrent " ,
" --no-warn-script-location " ,
" --user " ,
#'-t', target,
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
# "triton",
# "--index-url",
" https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl " ,
" --no-warn-script-location " ,
" --user " ,
#'-t', target,
]
)
else :
import_module ( self , " triton " , " triton " )
# subprocess.check_call(
# [
# pybin,
# #"-m",
# "pip",
# "uninstall",
# "mediapipe",
# #"--no-warn-script-location",
# #"--user",
# #'-t', target,
# ]
# )
# import_module(self, "mediapipe", "mediapipe")
# import_module(self, "audiocraft", "git+https://github.com/facebookresearch/audiocraft.git")
# subprocess.check_call([pybin,"-m","pip","install","force-reinstall","no-deps","pre xformers"])
subprocess . check_call ( [ pybin , " -m " , " pip " , " install " , " numpy " , " --upgrade " ] )
import_module ( self , " controlnet_aux " , " controlnet_aux " )
# import_module(self, "compel", "compel")
# import_module(self, "peft", "git+https://github.com/huggingface/peft.git")
subprocess . check_call ( [ pybin , " -m " , " pip " , " install " , " peft " , " --upgrade " ] )
# import_module(self, "diffusers", "git+https://github.com/huggingface/diffusers.git")
def get_module_dependencies ( module_name ) :
"""
Get the list of dependencies for a given module .
"""
# app_path = site.USER_SITE
# if app_path not in sys.path:
# sys.path.append(app_path)
# pybin = sys.executable
pybin = python_exec ( )
result = subprocess . run (
[ pybin , " -m " , " pip " , " show " , module_name ] , capture_output = True , text = True
)
output = result . stdout . strip ( )
dependencies = [ ]
for line in output . split ( " \n " ) :
if line . startswith ( " Requires: " ) :
dependencies = line . split ( " : " ) [ 1 ] . strip ( ) . split ( " , " )
break
return dependencies
def uninstall_module_with_dependencies ( module_name ) :
"""
Uninstall a module and its dependencies .
"""
show_system_console ( True )
set_system_console_topmost ( True )
# app_path = site.USER_SITE
# if app_path not in sys.path:
# sys.path.append(app_path)
# pybin = sys.executable
pybin = python_exec ( )
dependencies = get_module_dependencies ( module_name )
# Uninstall the module
subprocess . run ( [ pybin , " -m " , " pip " , " uninstall " , " -y " , module_name ] )
# Uninstall the dependencies
for dependency in dependencies :
subprocess . run ( [ pybin , " -m " , " pip " , " uninstall " , " -y " , dependency ] )
subprocess . check_call ( [ pybin , " -m " , " pip " , " install " , " numpy " ] )
class GENERATOR_OT_install ( Operator ) :
""" Install all dependencies """
bl_idname = " sequencer.install_generator "
bl_label = " Install Dependencies "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
install_modules ( self )
self . report (
{ " INFO " } ,
" Installation of dependencies is finished. " ,
)
return { " FINISHED " }
class GENERATOR_OT_uninstall ( Operator ) :
""" Uninstall all dependencies """
bl_idname = " sequencer.uninstall_generator "
bl_label = " Uninstall Dependencies "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
uninstall_module_with_dependencies ( " torch " )
uninstall_module_with_dependencies ( " torchvision " )
uninstall_module_with_dependencies ( " torchaudio " )
# if os_platform == "Darwin" or os_platform == "Linux":
# uninstall_module_with_dependencies("sox")
# else:
uninstall_module_with_dependencies ( " PySoundFile " )
uninstall_module_with_dependencies ( " diffusers " )
uninstall_module_with_dependencies ( " accelerate " )
uninstall_module_with_dependencies ( " transformers " )
uninstall_module_with_dependencies ( " sentencepiece " )
uninstall_module_with_dependencies ( " safetensors " )
uninstall_module_with_dependencies ( " opencv_python " )
uninstall_module_with_dependencies ( " scipy " )
uninstall_module_with_dependencies ( " IPython " )
uninstall_module_with_dependencies ( " bark " )
uninstall_module_with_dependencies ( " xformers " )
uninstall_module_with_dependencies ( " imageio " )
uninstall_module_with_dependencies ( " invisible-watermark " )
uninstall_module_with_dependencies ( " pillow " )
uninstall_module_with_dependencies ( " libtorrent " )
# uninstall_module_with_dependencies("compel")
uninstall_module_with_dependencies ( " triton " )
uninstall_module_with_dependencies ( " cv2 " )
# uninstall_module_with_dependencies("mediapipe")
self . report (
{ " INFO " } ,
" \n Remove AI Models manually: \n Linux and macOS: ~/.cache/huggingface/hub \n Windows: %u serprofile % .cache \\ huggingface \\ hub " ,
)
return { " FINISHED " }
def input_strips_updated ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
movie_model_card = addon_prefs . movie_model_card
image_model_card = addon_prefs . image_model_card
scene = context . scene
type = scene . generatorai_typeselect
input = scene . input_strips
if (
movie_model_card == " stabilityai/stable-diffusion-xl-base-1.0 "
and type == " movie "
) :
scene . input_strips = " input_strips "
if (
type == " movie "
or type == " audio "
or image_model_card == " lllyasviel/control_v11p_sd15_scribble "
) :
scene . inpaint_selected_strip = " "
if type == " image " and (
image_model_card == " lllyasviel/sd-controlnet-canny "
or image_model_card == " lllyasviel/sd-controlnet-openpose "
or image_model_card == " lllyasviel/control_v11p_sd15_scribble "
or image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
or image_model_card == " Salesforce/blipdiffusion "
) :
scene . input_strips = " input_strips "
if context . scene . lora_folder :
bpy . ops . lora . refresh_files ( )
if type == " text " :
scene . input_strips = " input_strips "
if (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid "
) or (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid-xt "
) :
scene . input_strips = " input_strips "
def output_strips_updated ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
movie_model_card = addon_prefs . movie_model_card
image_model_card = addon_prefs . image_model_card
scene = context . scene
type = scene . generatorai_typeselect
input = scene . input_strips
if (
type == " movie "
or type == " audio "
or image_model_card == " lllyasviel/control_v11p_sd15_scribble "
) :
scene . inpaint_selected_strip = " "
if context . scene . lora_folder :
bpy . ops . lora . refresh_files ( )
if (
image_model_card == " lllyasviel/sd-controlnet-canny "
or image_model_card == " lllyasviel/sd-controlnet-openpose "
or image_model_card == " lllyasviel/control_v11p_sd15_scribble "
or image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
or image_model_card == " Salesforce/blipdiffusion "
) and type == " image " :
scene . input_strips = " input_strips "
if type == " text " :
scene . input_strips = " input_strips "
if (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid "
) or (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid-xt "
) :
scene . input_strips = " input_strips "
class GeneratorAddonPreferences ( AddonPreferences ) :
bl_idname = __name__
soundselect : EnumProperty (
name = " Sound " ,
items = {
( " ding " , " Ding " , " A simple bell sound " ) ,
( " coin " , " Coin " , " A Mario-like coin sound " ) ,
( " user " , " User " , " Load a custom sound file " ) ,
} ,
default = " ding " ,
)
default_folder = os . path . join (
os . path . dirname ( os . path . abspath ( __file__ ) ) , " sounds " , " *.wav "
)
if default_folder not in sys . path :
sys . path . append ( default_folder )
usersound : StringProperty (
name = " User " ,
description = " Load a custom sound from your computer " ,
subtype = " FILE_PATH " ,
default = default_folder ,
maxlen = 1024 ,
)
playsound : BoolProperty (
name = " Audio Notification " ,
default = True ,
)
movie_model_card : bpy . props . EnumProperty (
name = " Video Model " ,
items = [
(
" stabilityai/stable-video-diffusion-img2vid-xt " ,
" Stable Video Diffusion XT (576x1024x24) " ,
" stabilityai/stable-video-diffusion-img2vid-xt " ,
) ,
(
" stabilityai/stable-video-diffusion-img2vid " ,
" Stable Video Diffusion (576x1024x14) " ,
" stabilityai/stable-video-diffusion-img2vid " ,
) ,
# (
# "stabilityai/stable-diffusion-xl-base-1.0",
# "Img2img SD XL 1.0 Refine (1024x1024)",
# "Stable Diffusion XL 1.0",
# ),
# (
# "stabilityai/sd-turbo",
# "Img2img SD Turbo (512x512)",
# "stabilityai/sd-turbo",
# ),
#("camenduru/potat1", "Potat v1 (1024x576)", "Potat (1024x576)"),
# ("VideoCrafter/Image2Video-512", "VideoCrafter v1 (512x512)", "VideoCrafter/Image2Video-512"),
(
" cerspense/zeroscope_v2_XL " ,
" Zeroscope XL (1024x576x24) " ,
" Zeroscope XL (1024x576x24) " ,
) ,
(
" cerspense/zeroscope_v2_576w " ,
" Zeroscope (576x320x24) " ,
" Zeroscope (576x320x24) " ,
) ,
(
" cerspense/zeroscope_v2_dark_30x448x256 " ,
" Zeroscope (448x256x30) " ,
" Zeroscope (448x256x30) " ,
) ,
(
" guoyww/animatediff-motion-adapter-v1-5-2 " ,
" AnimateDiff " ,
" AnimateDiff " ,
) ,
# ("hotshotco/Hotshot-XL", "Hotshot-XL (512x512)", "Hotshot-XL (512x512)"),
( " strangeman3107/animov-512x " , " Animov (512x512) " , " Animov (512x512) " ) ,
( " strangeman3107/animov-0.1.1 " , " Animov (448x384) " , " Animov (448x384) " ) ,
] ,
default = " cerspense/zeroscope_v2_576w " ,
update = input_strips_updated ,
)
image_model_card : bpy . props . EnumProperty (
name = " Image Model " ,
items = [
(
" stabilityai/stable-diffusion-xl-base-1.0 " ,
" Stable Diffusion XL 1.0 (1024x1024) " ,
" stabilityai/stable-diffusion-xl-base-1.0 " ,
) ,
(
" stabilityai/sdxl-turbo " ,
" Stable Diffusion XL Turbo (512x512) " ,
" stabilityai/sdxl-turbo " ,
) ,
(
" stabilityai/sd-turbo " ,
" Stable Diffusion Turbo (512x512) " ,
" stabilityai/sd-turbo " ,
) ,
(
" stabilityai/stable-diffusion-2 " ,
" Stable Diffusion 2 (768x768) " ,
" stabilityai/stable-diffusion-2 " ,
) ,
(
" runwayml/stable-diffusion-v1-5 " ,
" Stable Diffusion 1.5 (512x512) " ,
" runwayml/stable-diffusion-v1-5 " ,
) ,
(
" segmind/SSD-1B " ,
" Segmind SSD-1B (1024x1024) " ,
" segmind/SSD-1B " ,
) ,
(
" Lykon/dreamshaper-7 " ,
" Dreamshaper LCM v7 (1024 x 1024) " ,
" Lykon/dreamshaper-7 " ,
) ,
# ("ptx0/terminus-xl-gamma-v1", "Terminus XL Gamma v1", "ptx0/terminus-xl-gamma-v1"),
( " warp-ai/wuerstchen " , " Würstchen (1024x1024) " , " warp-ai/wuerstchen " ) ,
( " playgroundai/playground-v2-1024px-aesthetic " , " Playground v2 (1024x1024) " , " playgroundai/playground-v2-1024px-aesthetic " ) ,
(
" Salesforce/blipdiffusion " ,
" Blip Subject Driven (512x512) " ,
" Salesforce/blipdiffusion " ,
) ,
(
" lllyasviel/sd-controlnet-canny " ,
" ControlNet (512x512) " ,
" lllyasviel/sd-controlnet-canny " ,
) ,
#("DeepFloyd/IF-I-M-v1.0", "DeepFloyd/IF-I-M-v1.0", "DeepFloyd/IF-I-M-v1.0"),
(
" monster-labs/control_v1p_sd15_qrcode_monster " ,
" Illusion (512x512) " ,
" monster-labs/control_v1p_sd15_qrcode_monster " ,
) ,
(
" lllyasviel/sd-controlnet-openpose " ,
" OpenPose (512x512) " ,
" lllyasviel/sd-controlnet-openpose " ,
) ,
(
" lllyasviel/control_v11p_sd15_scribble " ,
" Scribble (512x512) " ,
" lllyasviel/control_v11p_sd15_scribble " ,
) ,
] ,
default = " stabilityai/stable-diffusion-xl-base-1.0 " ,
update = input_strips_updated ,
)
audio_model_card : bpy . props . EnumProperty (
name = " Audio Model " ,
items = [
(
" facebook/musicgen-stereo-small " ,
" Music: MusicGen Stereo " ,
" facebook/musicgen-stereo-small " ,
) ,
( " bark " , " Speech: Bark " , " Bark " ) ,
(
" cvssp/audioldm2-music " ,
" Music: AudioLDM 2 " ,
" Music: AudioLDM 2 " ,
) ,
(
" cvssp/audioldm2 " ,
" Sound: AudioLDM 2 " ,
" Sound: AudioLDM 2 " ,
) ,
# ("declare-lab/mustango", "Mustango", "declare-lab/mustango"),
] ,
default = " facebook/musicgen-stereo-small " ,
update = input_strips_updated ,
)
hugginface_token : bpy . props . StringProperty (
name = " Hugginface Token " ,
default = " hugginface_token " ,
subtype = " PASSWORD " ,
)
text_model_card : EnumProperty (
name = " Text Model " ,
items = {
(
" Salesforce/blip-image-captioning-large " ,
" Image Captioning " ,
" Salesforce/blip-image-captioning-large " ,
) ,
} ,
default = " Salesforce/blip-image-captioning-large " ,
)
generator_ai : StringProperty (
name = " Filepath " ,
description = " Path to the folder where the generated files are stored " ,
subtype = " DIR_PATH " ,
default = join ( bpy . utils . user_resource ( " DATAFILES " ) , " Generator AI " ) ,
)
use_strip_data : BoolProperty (
name = " Use Input Strip Data " ,
default = True ,
)
def draw ( self , context ) :
layout = self . layout
box = layout . box ( )
row = box . row ( )
row . operator ( " sequencer.install_generator " )
row . operator ( " sequencer.uninstall_generator " )
box . prop ( self , " movie_model_card " )
box . prop ( self , " image_model_card " )
if self . image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
row = box . row ( align = True )
row . prop ( self , " hugginface_token " )
row . operator (
" wm.url_open " , text = " " , icon = " URL "
) . url = " https://huggingface.co/settings/tokens "
box . prop ( self , " audio_model_card " )
box . prop ( self , " generator_ai " )
row = box . row ( align = True )
row . label ( text = " Notification: " )
row . prop ( self , " playsound " , text = " " )
sub_row = row . row ( )
sub_row . prop ( self , " soundselect " , text = " " )
if self . soundselect == " user " :
sub_row . prop ( self , " usersound " , text = " " )
sub_row . operator ( " renderreminder.play_notification " , text = " " , icon = " PLAY " )
sub_row . active = self . playsound
row_row = box . row ( align = True )
row_row . label ( text = " Use Input Strip Data: " )
row_row . prop ( self , " use_strip_data " , text = " " )
row_row . label ( text = " " )
row_row . label ( text = " " )
row_row . label ( text = " " )
class GENERATOR_OT_sound_notification ( Operator ) :
""" Test your notification settings """
bl_idname = " renderreminder.play_notification "
bl_label = " Test Notification "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
if addon_prefs . playsound :
device = aud . Device ( )
def coinSound ( ) :
sound = aud . Sound ( " " )
handle = device . play (
sound . triangle ( 1000 )
. highpass ( 20 )
. lowpass ( 2000 )
. ADSR ( 0 , 0.5 , 1 , 0 )
. fadeout ( 0.1 , 0.1 )
. limit ( 0 , 1 )
)
handle = device . play (
sound . triangle ( 1500 )
. highpass ( 20 )
. lowpass ( 2000 )
. ADSR ( 0 , 0.5 , 1 , 0 )
. fadeout ( 0.2 , 0.2 )
. delay ( 0.1 )
. limit ( 0 , 1 )
)
def ding ( ) :
sound = aud . Sound ( " " )
handle = device . play (
sound . triangle ( 3000 )
. highpass ( 20 )
. lowpass ( 1000 )
. ADSR ( 0 , 0.5 , 1 , 0 )
. fadeout ( 0 , 1 )
. limit ( 0 , 1 )
)
if addon_prefs . soundselect == " ding " :
ding ( )
if addon_prefs . soundselect == " coin " :
coinSound ( )
if addon_prefs . soundselect == " user " :
file = str ( addon_prefs . usersound )
if os . path . isfile ( file ) :
sound = aud . Sound ( file )
handle = device . play ( sound )
return { " FINISHED " }
def get_render_strip ( self , context , strip ) :
""" Render selected strip to hard disk """
# Check for the context and selected strips
if not context or not context . scene or not context . scene . sequence_editor :
self . report ( { " ERROR " } , " No valid context or selected strips " )
return { " CANCELLED " }
bpy . context . preferences . system . sequencer_proxy_setup = " MANUAL "
current_scene = context . scene
sequencer = current_scene . sequence_editor
current_frame_old = bpy . context . scene . frame_current
selected_sequences = strip
# Get the first empty channel above all strips
insert_channel_total = 1
for s in sequencer . sequences_all :
if s . channel > = insert_channel_total :
insert_channel_total = s . channel + 1
if strip . type in {
" MOVIE " ,
" IMAGE " ,
" SOUND " ,
" SCENE " ,
" TEXT " ,
" COLOR " ,
" META " ,
" MASK " ,
} :
# Deselect all strips in the current scene
for s in sequencer . sequences_all :
s . select = False
# Select the current strip in the current scene
strip . select = True
# Store current frame for later
bpy . context . scene . frame_current = int ( strip . frame_start )
# Copy the strip to the clipboard
bpy . ops . sequencer . copy ( )
# Create a new scene
# new_scene = bpy.data.scenes.new(name="New Scene")
# Create a new scene
new_scene = bpy . ops . scene . new ( type = " EMPTY " )
# Get the newly created scene
new_scene = bpy . context . scene
# Add a sequencer to the new scene
new_scene . sequence_editor_create ( )
# Set the new scene as the active scene
context . window . scene = new_scene
# Copy the scene properties from the current scene to the new scene
new_scene . render . resolution_x = current_scene . render . resolution_x
new_scene . render . resolution_y = current_scene . render . resolution_y
new_scene . render . resolution_percentage = (
current_scene . render . resolution_percentage
)
new_scene . render . pixel_aspect_x = current_scene . render . pixel_aspect_x
new_scene . render . pixel_aspect_y = current_scene . render . pixel_aspect_y
new_scene . render . fps = current_scene . render . fps
new_scene . render . fps_base = current_scene . render . fps_base
new_scene . render . sequencer_gl_preview = (
current_scene . render . sequencer_gl_preview
)
new_scene . render . use_sequencer_override_scene_strip = (
current_scene . render . use_sequencer_override_scene_strip
)
new_scene . world = current_scene . world
area = [
area for area in context . screen . areas if area . type == " SEQUENCE_EDITOR "
] [ 0 ]
with bpy . context . temp_override ( area = area ) :
# Paste the strip from the clipboard to the new scene
bpy . ops . sequencer . paste ( )
# Get the new strip in the new scene
new_strip = (
new_scene . sequence_editor . active_strip
) = bpy . context . selected_sequences [ 0 ]
# Set the range in the new scene to fit the pasted strip
new_scene . frame_start = int ( new_strip . frame_final_start )
new_scene . frame_end = (
int ( new_strip . frame_final_start + new_strip . frame_final_duration ) - 1
)
# Set the render settings for rendering animation with FFmpeg and MP4 with sound
bpy . context . scene . render . image_settings . file_format = " FFMPEG "
bpy . context . scene . render . ffmpeg . format = " MPEG4 "
bpy . context . scene . render . ffmpeg . audio_codec = " AAC "
# Make dir
preferences = bpy . context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
rendered_dir = os . path . join ( addon_prefs . generator_ai , str ( date . today ( ) ) )
rendered_dir = os . path . join ( rendered_dir , " Rendered_Strips " )
# Set the name of the file
src_name = strip . name
src_dir = " "
src_ext = " .mp4 "
# Create a new folder for the rendered files
if not os . path . exists ( rendered_dir ) :
os . makedirs ( rendered_dir )
# Set the output path for the rendering
output_path = os . path . join ( rendered_dir , src_name + " _rendered " + src_ext )
output_path = ensure_unique_filename ( output_path )
new_scene . render . filepath = output_path
# Render the strip to hard disk
bpy . ops . render . opengl ( animation = True , sequencer = True )
# Delete the new scene
bpy . data . scenes . remove ( new_scene , do_unlink = True )
if not os . path . exists ( output_path ) :
print ( " Render failed: " + output_path )
bpy . context . preferences . system . sequencer_proxy_setup = " AUTOMATIC "
return { " CANCELLED " }
# Set the original scene as the active scene
context . window . scene = current_scene
# Reset to total top channel
insert_channel = insert_channel_total
area = [
area for area in context . screen . areas if area . type == " SEQUENCE_EDITOR "
] [ 0 ]
with bpy . context . temp_override ( area = area ) :
insert_channel = find_first_empty_channel (
strip . frame_final_start ,
strip . frame_final_start + strip . frame_final_duration ,
)
if strip . type == " SOUND " :
# Insert the rendered file as a sound strip in the original scene without video.
bpy . ops . sequencer . sound_strip_add (
channel = insert_channel ,
filepath = output_path ,
frame_start = int ( strip . frame_final_start ) ,
overlap = 0 ,
)
elif strip . type == " SCENE " :
# Insert the rendered file as a scene strip in the original scene.
bpy . ops . sequencer . movie_strip_add (
channel = insert_channel ,
filepath = output_path ,
frame_start = int ( strip . frame_final_start ) ,
overlap = 0 ,
sound = False ,
)
# elif strip.type == "IMAGE":
# # Insert the rendered file as an image strip in the original scene.
# bpy.ops.sequencer.image_strip_add(
# channel=insert_channel,
# filepath=output_path,
# frame_start=int(strip.frame_final_start),
# overlap=0,
# sound=False,
# )
else :
# Insert the rendered file as a movie strip in the original scene without sound.
bpy . ops . sequencer . movie_strip_add (
channel = insert_channel ,
filepath = output_path ,
frame_start = int ( strip . frame_final_start ) ,
overlap = 0 ,
sound = False ,
)
resulting_strip = sequencer . active_strip
resulting_strip . use_proxy = False
# Reset current frame
bpy . context . scene . frame_current = current_frame_old
bpy . context . preferences . system . sequencer_proxy_setup = " AUTOMATIC "
return resulting_strip
def find_strip_by_name ( scene , name ) :
for sequence in scene . sequence_editor . sequences :
if sequence . name == name :
return sequence
return None
def get_strip_path ( strip ) :
if strip . type == " IMAGE " :
strip_dirname = os . path . dirname ( strip . directory )
image_path = bpy . path . abspath (
os . path . join ( strip_dirname , strip . elements [ 0 ] . filename )
)
return image_path
if strip . type == " MOVIE " :
movie_path = bpy . path . abspath ( strip . filepath )
return movie_path
return None
def clamp_value ( value , min_value , max_value ) :
# Ensure value is within the specified range
return max ( min ( value , max_value ) , min_value )
def find_overlapping_frame ( strip , current_frame ) :
# Calculate the end frame of the strip
strip_end_frame = strip . frame_final_start + strip . frame_duration
# Check if the strip's frame range overlaps with the current frame
if strip . frame_final_start < = current_frame < = strip_end_frame :
# Calculate the overlapped frame by subtracting strip.frame_start from the current frame
return current_frame - strip . frame_start
else :
return None # Return None if there is no overlap
def ensure_unique_filename ( file_name ) :
# Check if the file already exists
if os . path . exists ( file_name ) :
base_name , extension = os . path . splitext ( file_name )
index = 1
# Keep incrementing the index until a unique filename is found
while True :
unique_file_name = f " { base_name } _ { index } { extension } "
if not os . path . exists ( unique_file_name ) :
return unique_file_name
index + = 1
else :
# File doesn't exist, return the original name
return file_name
# LoRA.
class LORABrowserFileItem ( PropertyGroup ) :
name : bpy . props . StringProperty ( )
enabled : bpy . props . BoolProperty ( default = True )
weight_value : bpy . props . FloatProperty ( default = 1.0 )
index : bpy . props . IntProperty ( name = " Index " , default = 0 )
class LORABROWSER_UL_files ( UIList ) :
def draw_item (
self , context , layout , data , item , icon , active_data , active_propname , index
) :
row = layout . row ( align = True )
row . prop ( item , " enabled " , text = " " )
split = row . split ( factor = 0.7 )
split . label ( text = item . name )
split . prop ( item , " weight_value " , text = " " , emboss = False )
def update_folder_callback ( self , context ) :
if context . scene . lora_folder :
bpy . ops . lora . refresh_files ( )
class LORA_OT_RefreshFiles ( Operator ) :
bl_idname = " lora.refresh_files "
bl_label = " Refresh Files "
def execute ( self , context ) :
scene = context . scene
directory = bpy . path . abspath ( scene . lora_folder )
if not directory :
self . report ( { " ERROR " } , " No folder selected " )
return { " CANCELLED " }
lora_files = scene . lora_files
lora_files . clear ( )
for filename in os . listdir ( directory ) :
if filename . endswith ( " .safetensors " ) :
file_item = lora_files . add ( )
file_item . name = filename . replace ( " .safetensors " , " " )
file_item . enabled = False
file_item . weight_value = 1.0
return { " FINISHED " }
class SEQUENCER_PT_pallaidium_panel ( Panel ) : # UI
""" Generate Media using AI """
bl_idname = " SEQUENCER_PT_sequencer_generate_movie_panel "
bl_label = " Generative AI "
bl_space_type = " SEQUENCE_EDITOR "
bl_region_type = " UI "
bl_category = " Generative AI "
@classmethod
def poll ( cls , context ) :
return context . area . type == " SEQUENCE_EDITOR "
def draw ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
audio_model_card = addon_prefs . audio_model_card
movie_model_card = addon_prefs . movie_model_card
image_model_card = addon_prefs . image_model_card
scene = context . scene
type = scene . generatorai_typeselect
input = scene . input_strips
layout = self . layout
col = layout . column ( align = False )
col . use_property_split = True
col . use_property_decorate = False
col = col . box ( )
col = col . column ( )
# Input
if image_model_card == " Salesforce/blipdiffusion " and type == " image " :
col . prop ( context . scene , " input_strips " , text = " Source Image " )
col . prop ( context . scene , " blip_cond_subject " , text = " Source Subject " )
# col.prop(context.scene, "blip_subject_image", text="Target Image")
col . prop_search (
scene ,
" blip_subject_image " ,
scene . sequence_editor ,
" sequences " ,
text = " Target Image " ,
icon = " SEQ_STRIP_DUPLICATE " ,
)
col . prop ( context . scene , " blip_tgt_subject " , text = " Target Subject " )
else :
col . prop ( context . scene , " input_strips " , text = " Input " )
if type != " text " :
if type != " audio " :
if (
type == " movie "
and movie_model_card != " guoyww/animatediff-motion-adapter-v1-5-2 "
) or (
type == " image "
and image_model_card != " lllyasviel/sd-controlnet-canny "
and image_model_card != " lllyasviel/sd-controlnet-openpose "
and image_model_card != " lllyasviel/control_v11p_sd15_scribble "
and image_model_card
!= " monster-labs/control_v1p_sd15_qrcode_monster "
and image_model_card != " Salesforce/blipdiffusion "
) :
if input == " input_strips " and not scene . inpaint_selected_strip :
col = col . column ( heading = " Use " , align = True )
col . prop ( addon_prefs , " use_strip_data " , text = " Name & Seed " )
col . prop ( context . scene , " image_power " , text = " Strip Power " )
if (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid "
) or (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid-xt "
) :
col . prop (
context . scene , " svd_motion_bucket_id " , text = " Motion "
)
col . prop (
context . scene ,
" svd_decode_chunk_size " ,
text = " Decode Frames " ,
)
if bpy . context . scene . sequence_editor is not None :
if len ( bpy . context . scene . sequence_editor . sequences ) > 0 :
if input == " input_strips " and type == " image " :
col . prop_search (
scene ,
" inpaint_selected_strip " ,
scene . sequence_editor ,
" sequences " ,
text = " Inpaint Mask " ,
icon = " SEQ_STRIP_DUPLICATE " ,
)
if (
image_model_card == " lllyasviel/sd-controlnet-openpose "
and type == " image "
) :
col = col . column ( heading = " Read as " , align = True )
col . prop ( context . scene , " openpose_use_bones " , text = " OpenPose Rig Image " )
if (
image_model_card == " lllyasviel/control_v11p_sd15_scribble "
and type == " image "
) :
col = col . column ( heading = " Read as " , align = True )
col . prop ( context . scene , " use_scribble_image " , text = " Scribble Image " )
# LoRA.
if (
(
image_model_card == " stabilityai/stable-diffusion-xl-base-1.0 "
or image_model_card == " runwayml/stable-diffusion-v1-5 "
or image_model_card == " stabilityai/sdxl-turbo "
)
and type == " image "
and input != " input_strips "
) :
col = layout . column ( align = True )
col = col . box ( )
col = col . column ( align = True )
col . use_property_split = False
col . use_property_decorate = False
# Folder selection and refresh button
row = col . row ( align = True )
row . prop ( scene , " lora_folder " , text = " LoRA " )
row . operator ( " lora.refresh_files " , text = " " , icon = " FILE_REFRESH " )
# Custom UIList
lora_files = scene . lora_files
list_len = len ( lora_files )
if list_len > 0 :
col . template_list (
" LORABROWSER_UL_files " ,
" The_List " ,
scene ,
" lora_files " ,
scene ,
" lora_files_index " ,
rows = 2 ,
)
# Prompts
col = layout . column ( align = True )
col = col . box ( )
col = col . column ( align = True )
col . use_property_split = True
col . use_property_decorate = False
if (
type == " movie "
and movie_model_card == " stabilityai/stable-video-diffusion-img2vid "
) or (
type == " movie "
and movie_model_card == " stabilityai/stable-video-diffusion-img2vid-xt "
) :
pass
else :
col . use_property_split = False
col . use_property_decorate = False
col . prop ( context . scene , " generate_movie_prompt " , text = " " , icon = " ADD " )
if ( type == " audio " and audio_model_card == " bark " ) or (
type == " audio "
and audio_model_card == " facebook/musicgen-stereo-small "
) :
pass
else :
col . prop (
context . scene ,
" generate_movie_negative_prompt " ,
text = " " ,
icon = " REMOVE " ,
)
layout = col . column ( )
col = layout . column ( align = True )
col . use_property_split = True
col . use_property_decorate = False
if type != " audio " :
col . prop ( context . scene , " generatorai_styles " , text = " Style " )
layout = col . column ( )
if type == " movie " or type == " image " :
col = layout . column ( align = True )
col . prop ( context . scene , " generate_movie_x " , text = " X " )
col . prop ( context . scene , " generate_movie_y " , text = " Y " )
col = layout . column ( align = True )
if type == " movie " or type == " image " :
col . prop ( context . scene , " generate_movie_frames " , text = " Frames " )
if type == " audio " and audio_model_card != " bark " :
col . prop ( context . scene , " audio_length_in_f " , text = " Frames " )
if type == " audio " and audio_model_card == " bark " :
col = layout . column ( align = True )
col . prop ( context . scene , " speakers " , text = " Speaker " )
col . prop ( context . scene , " languages " , text = " Language " )
elif (
type == " audio "
and addon_prefs . audio_model_card == " facebook/musicgen-stereo-small "
) :
col . prop (
context . scene , " movie_num_inference_steps " , text = " Quality Steps "
)
else :
col . prop (
context . scene , " movie_num_inference_steps " , text = " Quality Steps "
)
if (
(
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid "
)
or (
type == " movie "
and movie_model_card
== " stabilityai/stable-video-diffusion-img2vid-xt "
)
) :
pass
else :
col . prop ( context . scene , " movie_num_guidance " , text = " Word Power " )
col = col . column ( )
row = col . row ( align = True )
sub_row = row . row ( align = True )
sub_row . prop ( context . scene , " movie_num_seed " , text = " Seed " )
row . prop ( context . scene , " movie_use_random " , text = " " , icon = " QUESTION " )
sub_row . active = not context . scene . movie_use_random
if type == " movie " and (
movie_model_card == " cerspense/zeroscope_v2_dark_30x448x256 "
or movie_model_card == " cerspense/zeroscope_v2_576w "
or movie_model_card == " cerspense/zeroscope_v2_XL "
) :
col = col . column ( heading = " Upscale " , align = True )
col . prop ( context . scene , " video_to_video " , text = " 2x " )
if type == " image " :
col = col . column ( heading = " Enhance " , align = True )
col . prop ( context . scene , " refine_sd " , text = " SD Refine " )
sub_col = col . row ( )
sub_col . active = context . scene . refine_sd
if type != " audio " :
row = col . row ( )
if type == " movie " or (
type == " image "
and image_model_card != " lllyasviel/sd-controlnet-canny "
and image_model_card != " lllyasviel/sd-controlnet-openpose "
and image_model_card != " lllyasviel/control_v11p_sd15_scribble "
and image_model_card
!= " monster-labs/control_v1p_sd15_qrcode_monster "
and image_model_card != " Salesforce/blipdiffusion "
) :
row . prop ( context . scene , " use_freeU " , text = " FreeU " )
if type == " image " :
if (
(
type == " image "
and image_model_card
== " stabilityai/stable-diffusion-xl-base-1.0 "
)
or ( type == " image " and image_model_card == " segmind/SSD-1B " )
or (
type == " image "
and image_model_card == " runwayml/stable-diffusion-v1-5 "
)
) :
row . prop ( context . scene , " use_lcm " , text = " LCM " )
# Output.
layout = self . layout
layout . use_property_split = True
layout . use_property_decorate = False
col = layout . box ( )
col = col . column ( align = True )
col . prop ( context . scene , " generatorai_typeselect " , text = " Output " )
if type == " image " :
col . prop ( addon_prefs , " image_model_card " , text = " " )
if addon_prefs . image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
row = col . row ( align = True )
row . prop ( addon_prefs , " hugginface_token " )
row . operator (
" wm.url_open " , text = " " , icon = " URL "
) . url = " https://huggingface.co/settings/tokens "
if type == " movie " :
col . prop ( addon_prefs , " movie_model_card " , text = " " )
if type == " audio " :
col . prop ( addon_prefs , " audio_model_card " , text = " " )
if type == " text " :
col . prop ( addon_prefs , " text_model_card " , text = " " )
if type != " text " :
col = col . column ( )
col . prop ( context . scene , " movie_num_batch " , text = " Batch Count " )
# Generate.
col = layout . column ( )
col = col . box ( )
if input == " input_strips " :
ed = scene . sequence_editor
row = col . row ( align = True )
row . scale_y = 1.2
row . operator ( " sequencer.text_to_generator " , text = " Generate from Strips " )
else :
row = col . row ( align = True )
row . scale_y = 1.2
if type == " movie " :
# if movie_model_card == "stabilityai/stable-diffusion-xl-base-1.0":
# row.operator(
# "sequencer.text_to_generator", text="Generate from Strips"
# )
# else:
if movie_model_card == " stabilityai/sd-turbo " :
row . operator (
" sequencer.text_to_generator " , text = " Generate from Strips "
)
else :
row . operator ( " sequencer.generate_movie " , text = " Generate " )
if type == " image " :
row . operator ( " sequencer.generate_image " , text = " Generate " )
if type == " audio " :
row . operator ( " sequencer.generate_audio " , text = " Generate " )
class NoWatermark :
def apply_watermark ( self , img ) :
return img
class SEQUENCER_OT_generate_movie ( Operator ) :
""" Generate Video """
bl_idname = " sequencer.generate_movie "
bl_label = " Prompt "
bl_description = " Convert text to video "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
if not scene . generate_movie_prompt :
self . report ( { " INFO " } , " Text prompt in the Generative AI tab is empty! " )
return { " CANCELLED " }
try :
import torch
from diffusers . utils import export_to_video
from PIL import Image
Image . MAX_IMAGE_PIXELS = None
import numpy as np
from . free_lunch_utils import (
register_free_upblock3d ,
register_free_crossattn_upblock3d ,
)
# from compel import Compel
except ModuleNotFoundError :
print ( " In the add-on preferences, install dependencies. " )
self . report (
{ " INFO " } ,
" In the add-on preferences, install dependencies. " ,
)
return { " CANCELLED " }
show_system_console ( True )
set_system_console_topmost ( True )
seq_editor = scene . sequence_editor
if not seq_editor :
scene . sequence_editor_create ( )
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
current_frame = scene . frame_current
prompt = style_prompt ( scene . generate_movie_prompt ) [ 0 ]
negative_prompt = (
scene . generate_movie_negative_prompt
+ " , "
+ style_prompt ( scene . generate_movie_prompt ) [ 1 ]
+ " , nsfw, nude, nudity "
)
movie_x = scene . generate_movie_x
movie_y = scene . generate_movie_y
x = scene . generate_movie_x = closest_divisible_32 ( movie_x )
y = scene . generate_movie_y = closest_divisible_32 ( movie_y )
duration = scene . generate_movie_frames
movie_num_inference_steps = scene . movie_num_inference_steps
movie_num_guidance = scene . movie_num_guidance
input = scene . input_strips
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
movie_model_card = addon_prefs . movie_model_card
image_model_card = addon_prefs . image_model_card
pipe = None
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
# LOADING MODELS
print ( " Model: " + movie_model_card )
# Models for refine imported image or movie
if (
( scene . movie_path or scene . image_path )
and input == " input_strips "
and movie_model_card != " guoyww/animatediff-motion-adapter-v1-5-2 "
) :
if (
movie_model_card == " stabilityai/sd-turbo "
) : # img2img
# from diffusers import StableDiffusionXLImg2ImgPipeline, AutoencoderKL
# vae = AutoencoderKL.from_pretrained(
# "madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16
# )
from diffusers import AutoPipelineForImage2Image
#from diffusers.utils import load_image
pipe = AutoPipelineForImage2Image . from_pretrained ( " stabilityai/sd-turbo " , torch_dtype = torch . float16 , variant = " fp16 " )
#pipe.to("cuda")
#init_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png").resize((512, 512))
#prompt = "cat wizard, gandalf, lord of the rings, detailed, fantasy, cute, adorable, Pixar, Disney, 8k"
#image = pipe(prompt, image=init_image, num_inference_steps=2, strength=0.5, guidance_scale=0.0).images[0]
# pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(
# movie_model_card,
# torch_dtype=torch.float16,
# variant="fp16",
# vae=vae,
# )
# from diffusers import DPMSolverMultistepScheduler
# pipe.scheduler = DPMSolverMultistepScheduler.from_config(
# pipe.scheduler.config
# )
# pipe.watermark = NoWatermark()
if low_vram ( ) :
pipe . enable_model_cpu_offload ( )
# pipe.unet.enable_forward_chunking(chunk_size=1, dim=1) # Heavy
# pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
# from diffusers import StableDiffusionXLImg2ImgPipeline
# refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained(
# "stabilityai/stable-diffusion-xl-refiner-1.0",
# text_encoder_2=pipe.text_encoder_2,
# vae=pipe.vae,
# torch_dtype=torch.float16,
# variant="fp16",
# )
# if low_vram():
# refiner.enable_model_cpu_offload()
# # refiner.enable_vae_tiling()
# # refiner.enable_vae_slicing()
# else:
# refiner.to(gfx_device)
# if (
# movie_model_card == "stabilityai/stable-diffusion-xl-base-1.0"
# ): # img2img
# from diffusers import StableDiffusionXLImg2ImgPipeline, AutoencoderKL
# vae = AutoencoderKL.from_pretrained(
# "madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16
# )
# pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(
# movie_model_card,
# torch_dtype=torch.float16,
# variant="fp16",
# vae=vae,
# )
# from diffusers import DPMSolverMultistepScheduler
# pipe.scheduler = DPMSolverMultistepScheduler.from_config(
# pipe.scheduler.config
# )
# pipe.watermark = NoWatermark()
# if low_vram():
# pipe.enable_model_cpu_offload()
# # pipe.unet.enable_forward_chunking(chunk_size=1, dim=1) # Heavy
# # pipe.enable_vae_slicing()
# else:
# pipe.to(gfx_device)
# from diffusers import StableDiffusionXLImg2ImgPipeline
# refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained(
# "stabilityai/stable-diffusion-xl-refiner-1.0",
# text_encoder_2=pipe.text_encoder_2,
# vae=pipe.vae,
# torch_dtype=torch.float16,
# variant="fp16",
# )
# if low_vram():
# refiner.enable_model_cpu_offload()
# # refiner.enable_vae_tiling()
# # refiner.enable_vae_slicing()
# else:
# refiner.to(gfx_device)
elif (
movie_model_card == " stabilityai/stable-video-diffusion-img2vid "
or movie_model_card == " stabilityai/stable-video-diffusion-img2vid-xt "
) :
from diffusers import StableVideoDiffusionPipeline
from diffusers . utils import load_image , export_to_video
refiner = StableVideoDiffusionPipeline . from_pretrained (
movie_model_card , torch_dtype = torch . float16 , variant = " fp16 "
)
if low_vram ( ) :
refiner . enable_model_cpu_offload ( )
else :
refiner . to ( gfx_device )
else : # vid2vid / img2vid
if (
movie_model_card == " cerspense/zeroscope_v2_dark_30x448x256 "
or movie_model_card == " cerspense/zeroscope_v2_576w "
or scene . image_path
) :
card = " cerspense/zeroscope_v2_XL "
else :
card = movie_model_card
from diffusers import VideoToVideoSDPipeline
upscale = VideoToVideoSDPipeline . from_pretrained (
card ,
torch_dtype = torch . float16 ,
# variant="fp16",
# use_safetensors=True,
)
from diffusers import DPMSolverMultistepScheduler
upscale . scheduler = DPMSolverMultistepScheduler . from_config (
upscale . scheduler . config
)
if low_vram ( ) :
# torch.cuda.set_per_process_memory_fraction(0.98)
upscale . enable_model_cpu_offload ( )
# upscale.enable_vae_tiling()
# upscale.enable_vae_slicing()
#upscale.unet.enable_forward_chunking(chunk_size=1, dim=1) # heavy:
else :
upscale . to ( gfx_device )
# Models for movie generation
else :
if movie_model_card == " guoyww/animatediff-motion-adapter-v1-5-2 " :
from diffusers import MotionAdapter , AnimateDiffPipeline , DDIMScheduler
from diffusers . utils import export_to_gif
# Load the motion adapter
adapter = MotionAdapter . from_pretrained (
" guoyww/animatediff-motion-adapter-v1-5-2 "
)
# load SD 1.5 based finetuned model
# model_id = "runwayml/stable-diffusion-v1-5"
model_id = " SG161222/Realistic_Vision_V5.1_noVAE "
pipe = AnimateDiffPipeline . from_pretrained (
model_id , motion_adapter = adapter , torch_dtype = torch . float16
)
scheduler = DDIMScheduler . from_pretrained (
model_id ,
subfolder = " scheduler " ,
clip_sample = False ,
timestep_spacing = " linspace " ,
steps_offset = 1 ,
)
pipe . scheduler = scheduler
if low_vram ( ) :
pipe . enable_vae_slicing ( )
pipe . enable_model_cpu_offload ( )
# pipe.unet.enable_forward_chunking(chunk_size=1, dim=1) # heavy:
else :
upscale . to ( gfx_device )
elif movie_model_card == " VideoCrafter/Image2Video-512 " :
from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline . from_single_file (
" https://huggingface.co/VideoCrafter/Image2Video-512/blob/main/model.ckpt " ,
torch_dtype = torch . float16 ,
)
from diffusers import DPMSolverMultistepScheduler
pipe . scheduler = DPMSolverMultistepScheduler . from_config (
pipe . scheduler . config
)
if low_vram ( ) :
pipe . enable_model_cpu_offload ( )
# pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
elif (
movie_model_card == " stabilityai/stable-video-diffusion-img2vid "
or movie_model_card == " stabilityai/stable-video-diffusion-img2vid-xt "
) :
print ( " Stable Video Diffusion needs image input " )
return { " CANCELLED " }
else :
from diffusers import TextToVideoSDPipeline
import torch
pipe = TextToVideoSDPipeline . from_pretrained (
movie_model_card ,
torch_dtype = torch . float16 ,
use_safetensors = False ,
)
from diffusers import DPMSolverMultistepScheduler
pipe . scheduler = DPMSolverMultistepScheduler . from_config (
pipe . scheduler . config
)
if low_vram ( ) :
pipe . enable_model_cpu_offload ( )
# pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
# Model for upscale generated movie
if scene . video_to_video :
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
from diffusers import DiffusionPipeline
upscale = DiffusionPipeline . from_pretrained (
" cerspense/zeroscope_v2_XL " ,
torch_dtype = torch . float16 ,
use_safetensors = False ,
)
upscale . scheduler = DPMSolverMultistepScheduler . from_config (
upscale . scheduler . config
)
if low_vram ( ) :
upscale . enable_model_cpu_offload ( )
#upscale.unet.enable_forward_chunking(chunk_size=1, dim=1) # Heavy
# upscale.enable_vae_slicing()
else :
upscale . to ( gfx_device )
if scene . use_freeU and pipe : # Free Lunch
# -------- freeu block registration
print ( " Process: FreeU " )
register_free_upblock3d ( pipe ) # , b1=1.1, b2=1.2, s1=0.6, s2=0.4)
register_free_crossattn_upblock3d ( pipe ) # , b1=1.1, b2=1.2, s1=0.6, s2=0.4)
# -------- freeu block registration
# GENERATING - Main Loop
for i in range ( scene . movie_num_batch ) :
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
if i > 0 :
empty_channel = scene . sequence_editor . active_strip . channel
start_frame = (
scene . sequence_editor . active_strip . frame_final_start
+ scene . sequence_editor . active_strip . frame_final_duration
)
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
else :
empty_channel = find_first_empty_channel (
scene . frame_current ,
( scene . movie_num_batch * duration ) + scene . frame_current ,
)
start_frame = scene . frame_current
# Get seed
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( - 2147483647 , 2147483647 )
)
print ( " Seed: " + str ( seed ) )
context . scene . movie_num_seed = seed
# Use cuda if possible
if (
torch . cuda . is_available ( )
and movie_model_card != " stabilityai/stable-video-diffusion-img2vid "
and movie_model_card != " stabilityai/stable-video-diffusion-img2vid-xt "
) :
generator = (
torch . Generator ( " cuda " ) . manual_seed ( seed ) if seed != 0 else None
)
else :
if seed != 0 :
generator = torch . Generator ( )
generator . manual_seed ( seed )
else :
generator = None
# Process batch input
if (
( scene . movie_path or scene . image_path )
and input == " input_strips "
and movie_model_card != " guoyww/animatediff-motion-adapter-v1-5-2 "
) :
video_path = scene . movie_path
# # img2img
# if movie_model_card == "stabilityai/stable-diffusion-xl-base-1.0":
# print("Process: Frame by frame (SD XL)")
# input_video_path = video_path
# output_video_path = solve_path("temp_images")
# if scene.movie_path:
# frames = process_video(input_video_path, output_video_path)
# elif scene.image_path:
# frames = process_image(
# scene.image_path, int(scene.generate_movie_frames)
# )
# video_frames = []
# # Iterate through the frames
# for frame_idx, frame in enumerate(
# frames
# ): # would love to get this flicker free
# print(str(frame_idx + 1) + "/" + str(len(frames)))
# image = refiner(
# prompt,
# negative_prompt=negative_prompt,
# num_inference_steps=movie_num_inference_steps,
# strength=1.00 - scene.image_power,
# guidance_scale=movie_num_guidance,
# image=frame,
# generator=generator,
# ).images[0]
# video_frames.append(image)
# if torch.cuda.is_available():
# torch.cuda.empty_cache()
# video_frames = np.array(video_frames)
# img2img
if movie_model_card == " stabilityai/sd-turbo " :
print ( " Process: Frame by frame (SD Turbo) " )
input_video_path = video_path
output_video_path = solve_path ( " temp_images " )
if scene . movie_path :
frames = process_video ( input_video_path , output_video_path )
elif scene . image_path :
frames = process_image (
scene . image_path , int ( scene . generate_movie_frames )
)
video_frames = [ ]
# Iterate through the frames
for frame_idx , frame in enumerate ( frames ) : # would love to get this flicker free
print ( str ( frame_idx + 1 ) + " / " + str ( len ( frames ) ) )
image = pipe (
prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = 2 , #movie_num_inference_steps,
strength = 0.5 , #scene.image_power,
guidance_scale = 3.0 ,
image = frame ,
generator = generator ,
) . images [ 0 ]
video_frames . append ( image )
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
video_frames = np . array ( video_frames )
# vid2vid / img2vid
elif (
movie_model_card == " stabilityai/stable-video-diffusion-img2vid "
or movie_model_card
== " stabilityai/stable-video-diffusion-img2vid-xt "
) :
if scene . movie_path :
print ( " Only image strip as input supported! " )
return { " CANCELLED " }
elif scene . image_path :
print ( " Process: Image to SVD Video " )
if not os . path . isfile ( scene . image_path ) :
print ( " No file found. " )
return { " CANCELLED " }
image = load_image ( bpy . path . abspath ( scene . image_path ) )
image = image . resize (
( closest_divisible_32 ( int ( x ) ) , closest_divisible_32 ( int ( y ) ) )
)
video_frames = refiner (
image ,
noise_aug_strength = 1.00 - scene . image_power ,
decode_chunk_size = scene . svd_decode_chunk_size ,
motion_bucket_id = scene . svd_motion_bucket_id ,
num_inference_steps = movie_num_inference_steps ,
height = y ,
width = x ,
num_frames = duration ,
generator = generator ,
) . frames [ 0 ]
export_to_video (
video_frames , " C:/Users/45239/Documents/generated2.mp4 " , fps = 7
)
elif movie_model_card != " guoyww/animatediff-motion-adapter-v1-5-2 " :
if scene . movie_path :
print ( " Process: Video to video " )
if not os . path . isfile ( scene . movie_path ) :
print ( " No file found. " )
return { " CANCELLED " }
video = load_video_as_np_array ( video_path )
elif scene . image_path :
print ( " Process: Image to video " )
if not os . path . isfile ( scene . image_path ) :
print ( " No file found. " )
return { " CANCELLED " }
video = process_image (
scene . image_path , int ( scene . generate_movie_frames )
)
video = np . array ( video )
if not video . any ( ) :
print ( " Loading of file failed " )
return { " CANCELLED " }
# Upscale video
if scene . video_to_video :
video = [
Image . fromarray ( frame ) . resize (
(
closest_divisible_32 ( int ( x * 2 ) ) ,
closest_divisible_32 ( int ( y * 2 ) ) ,
)
)
for frame in video
]
else :
video = [
Image . fromarray ( frame ) . resize (
(
closest_divisible_32 ( int ( x ) ) ,
closest_divisible_32 ( int ( y ) ) ,
)
)
for frame in video
]
video_frames = upscale (
prompt ,
video = video ,
strength = 1.00 - scene . image_power ,
negative_prompt = negative_prompt ,
num_inference_steps = movie_num_inference_steps ,
guidance_scale = movie_num_guidance ,
generator = generator ,
) . frames
# Movie.
else :
print ( " Generate: Video " )
if movie_model_card == " guoyww/animatediff-motion-adapter-v1-5-2 " :
output = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = movie_num_inference_steps ,
guidance_scale = movie_num_guidance ,
height = y ,
width = x ,
num_frames = duration ,
generator = generator ,
)
video_frames = output . frames [ 0 ]
else :
video_frames = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = movie_num_inference_steps ,
guidance_scale = movie_num_guidance ,
height = y ,
width = x ,
num_frames = duration ,
generator = generator ,
) . frames
movie_model_card = addon_prefs . movie_model_card
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
# Upscale video.
if scene . video_to_video :
print ( " Upscale: Video " )
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
video = [
Image . fromarray ( frame ) . resize (
( closest_divisible_32 ( x * 2 ) , closest_divisible_32 ( y * 2 ) )
)
for frame in video_frames
]
video_frames = upscale (
prompt ,
video = video ,
strength = 1.00 - scene . image_power ,
negative_prompt = negative_prompt ,
num_inference_steps = movie_num_inference_steps ,
guidance_scale = movie_num_guidance ,
generator = generator ,
) . frames
if movie_model_card == " guoyww/animatediff-motion-adapter-v1-5-2 " :
# from diffusers.utils import export_to_video
# Move to folder.
video_frames = np . array ( video_frames )
src_path = export_to_video ( video_frames )
dst_path = solve_path ( clean_filename ( str ( seed ) + " _ " + prompt ) + " .mp4 " )
shutil . move ( src_path , dst_path )
else :
# Move to folder.
src_path = export_to_video ( video_frames )
dst_path = solve_path ( clean_filename ( str ( seed ) + " _ " + prompt ) + " .mp4 " )
shutil . move ( src_path , dst_path )
# Add strip.
if not os . path . isfile ( dst_path ) :
print ( " No resulting file found. " )
return { " CANCELLED " }
for window in bpy . context . window_manager . windows :
screen = window . screen
for area in screen . areas :
if area . type == " SEQUENCE_EDITOR " :
from bpy import context
with context . temp_override ( window = window , area = area ) :
bpy . ops . sequencer . movie_strip_add (
filepath = dst_path ,
frame_start = start_frame ,
channel = empty_channel ,
fit_method = " FIT " ,
adjust_playback_rate = False ,
sound = False ,
use_framerate = False ,
)
strip = scene . sequence_editor . active_strip
strip . transform . filter = " SUBSAMPLING_3x3 "
scene . sequence_editor . active_strip = strip
strip . name = str ( seed ) + " _ " + prompt
strip . use_proxy = True
bpy . ops . sequencer . rebuild_proxy ( )
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip. Remove this if Blender crashes: https://docs.blender.org/api/current/info_gotcha.html#can-i-redraw-during-script-execution
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
break
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
bpy . types . Scene . movie_path = " "
bpy . ops . renderreminder . play_notification ( )
scene . frame_current = current_frame
return { " FINISHED " }
class SEQUENCER_OT_generate_audio ( Operator ) :
""" Generate Audio """
bl_idname = " sequencer.generate_audio "
bl_label = " Prompt "
bl_description = " Convert text to audio "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
if not scene . generate_movie_prompt :
self . report ( { " INFO " } , " Text prompt in the Generative AI tab is empty! " )
return { " CANCELLED " }
if not scene . sequence_editor :
scene . sequence_editor_create ( )
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
current_frame = scene . frame_current
prompt = scene . generate_movie_prompt
negative_prompt = scene . generate_movie_negative_prompt
movie_num_inference_steps = scene . movie_num_inference_steps
movie_num_guidance = scene . movie_num_guidance
audio_length_in_s = scene . audio_length_in_f / (
scene . render . fps / scene . render . fps_base
)
try :
import torch
import scipy
from scipy . io . wavfile import write as write_wav
if (
addon_prefs . audio_model_card == " cvssp/audioldm2 "
or addon_prefs . audio_model_card == " cvssp/audioldm2-music "
) :
from diffusers import AudioLDM2Pipeline , DPMSolverMultistepScheduler
import scipy
from IPython . display import Audio
import xformers
if addon_prefs . audio_model_card == " facebook/musicgen-stereo-small " :
# if os_platform == "Darwin" or os_platform == "Linux":
# import sox
# else:
import soundfile as sf
if addon_prefs . audio_model_card == " bark " :
os . environ [ " CUDA_VISIBLE_DEVICES " ] = " 0 "
import numpy as np
from bark . generation import (
generate_text_semantic ,
preload_models ,
)
from bark . api import semantic_to_waveform
from bark import generate_audio , SAMPLE_RATE
except ModuleNotFoundError :
print ( " Dependencies needs to be installed in the add-on preferences. " )
self . report (
{ " INFO " } ,
" Dependencies needs to be installed in the add-on preferences. " ,
)
return { " CANCELLED " }
show_system_console ( True )
set_system_console_topmost ( True )
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
print ( " Model: " + addon_prefs . audio_model_card )
# Load models
if (
addon_prefs . audio_model_card == " cvssp/audioldm2 "
or addon_prefs . audio_model_card == " cvssp/audioldm2-music "
) :
repo_id = addon_prefs . audio_model_card
pipe = AudioLDM2Pipeline . from_pretrained ( repo_id )
pipe . scheduler = DPMSolverMultistepScheduler . from_config (
pipe . scheduler . config
)
if low_vram ( ) :
pipe . enable_model_cpu_offload ( )
# pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
# Musicgen
elif addon_prefs . audio_model_card == " facebook/musicgen-stereo-small " :
from transformers import pipeline
from transformers import set_seed
pipe = pipeline (
" text-to-audio " ,
" facebook/musicgen-stereo-small " ,
device = " cuda:0 " ,
torch_dtype = torch . float16 ,
)
if int ( audio_length_in_s * 50 ) > 1503 :
self . report ( { " INFO " } , " Maximum output duration is 30 sec. " )
# Bark
elif addon_prefs . audio_model_card == " bark " :
preload_models (
text_use_small = True ,
coarse_use_small = True ,
fine_use_gpu = True ,
fine_use_small = True ,
)
# Mustango
elif addon_prefs . audio_model_card == " declare-lab/mustango " :
import IPython
import soundfile as sf
from diffusers import DiffusionPipeline
# from mustango import Mustango
# from transformers import pipeline
# from transformers import set_seed
model = DiffusionPipeline . from_pretrained (
" declare-lab/mustango "
) # , device="cuda:0", torch_dtype=torch.float16)
# Deadend
else :
print ( " Audio model not found. " )
self . report ( { " INFO " } , " Audio model not found. " )
return { " CANCELLED " }
# Main loop
for i in range ( scene . movie_num_batch ) :
if i > 0 :
empty_channel = scene . sequence_editor . active_strip . channel
start_frame = (
scene . sequence_editor . active_strip . frame_final_start
+ scene . sequence_editor . active_strip . frame_final_duration
)
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
else :
empty_channel = find_first_empty_channel (
scene . frame_current ,
100000000000000000000 ,
)
start_frame = scene . frame_current
if addon_prefs . audio_model_card == " bark " :
print ( " Generate: Speech (Bark) " )
rate = 24000
GEN_TEMP = 0.6
SPEAKER = " v2/ " + scene . languages + " _ " + scene . speakers
silence = np . zeros ( int ( 0.25 * rate ) ) # quarter second of silence
prompt = context . scene . generate_movie_prompt
prompt = prompt . replace ( " \n " , " " ) . strip ( )
sentences = split_and_recombine_text (
prompt , desired_length = 90 , max_length = 150
)
pieces = [ ]
for sentence in sentences :
print ( " Sentence: " + sentence )
semantic_tokens = generate_text_semantic (
sentence ,
history_prompt = SPEAKER ,
temp = GEN_TEMP ,
# min_eos_p=0.1, # this controls how likely the generation is to end
)
audio_array = semantic_to_waveform (
semantic_tokens , history_prompt = SPEAKER
)
pieces + = [ audio_array , silence . copy ( ) ]
audio = np . concatenate ( pieces )
filename = solve_path ( clean_filename ( prompt + " .wav " ) )
# Write the combined audio to a file
write_wav ( filename , rate , audio . transpose ( ) )
# Musicgen
elif addon_prefs . audio_model_card == " facebook/musicgen-stereo-small " :
print ( " Generate: MusicGen Stereo " )
print ( " Prompt: " + prompt )
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( 0 , 999999 )
)
print ( " Seed: " + str ( seed ) )
context . scene . movie_num_seed = seed
set_seed ( seed )
music = pipe (
prompt ,
forward_params = {
" max_new_tokens " : int ( min ( audio_length_in_s * 50 , 1503 ) )
} ,
)
filename = solve_path ( clean_filename ( str ( seed ) + " _ " + prompt ) + " .wav " )
rate = 48000
# if os_platform == "Darwin" or os_platform == "Linux":
# tfm = sox.Transformer()
# tfm.build_file(
# input_array=music["audio"][0].T,
# sample_rate_in=music["sampling_rate"],
# output_filepath=filename
# )
# else:
sf . write ( filename , music [ " audio " ] [ 0 ] . T , music [ " sampling_rate " ] )
elif addon_prefs . audio_model_card == " declare-lab/mustango " :
music = model . generate ( prompt )
sf . write ( filename , audio , samplerate = 16000 )
IPython . display . Audio ( data = audio , rate = 16000 )
else : # AudioLDM
print ( " Generate: Audio/music (AudioLDM) " )
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( 0 , 999999 )
)
print ( " Seed: " + str ( seed ) )
context . scene . movie_num_seed = seed
# Use cuda if possible
if torch . cuda . is_available ( ) :
generator = (
torch . Generator ( " cuda " ) . manual_seed ( seed ) if seed != 0 else None
)
else :
if seed != 0 :
generator = torch . Generator ( )
generator . manual_seed ( seed )
else :
generator = None
prompt = context . scene . generate_movie_prompt
print ( " Prompt: " + prompt )
audio = pipe (
prompt ,
num_inference_steps = movie_num_inference_steps ,
audio_length_in_s = audio_length_in_s ,
guidance_scale = movie_num_guidance ,
generator = generator ,
) . audios [ 0 ]
rate = 16000
filename = solve_path ( str ( seed ) + " _ " + prompt + " .wav " )
write_wav ( filename , rate , audio . transpose ( ) )
filepath = filename
if os . path . isfile ( filepath ) :
empty_channel = find_first_empty_channel (
start_frame , start_frame + scene . audio_length_in_f
)
strip = scene . sequence_editor . sequences . new_sound (
name = prompt ,
filepath = filepath ,
channel = empty_channel ,
frame_start = start_frame ,
)
scene . sequence_editor . active_strip = strip
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip. Remove this if Blender crashes:
# https://docs.blender.org/api/current/info_gotcha.html#can-i-redraw-during-script-execution
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
else :
print ( " No resulting file found! " )
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
bpy . ops . renderreminder . play_notification ( )
return { " FINISHED " }
class SEQUENCER_OT_generate_image ( Operator ) :
""" Generate Image """
bl_idname = " sequencer.generate_image "
bl_label = " Prompt "
bl_description = " Convert text to image "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
seq_editor = scene . sequence_editor
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
image_model_card = addon_prefs . image_model_card
strips = context . selected_sequences
type = scene . generatorai_typeselect
use_strip_data = addon_prefs . use_strip_data
pipe = None
refiner = None
converter = None
guidance = scene . movie_num_guidance
if (
scene . generate_movie_prompt == " "
and not image_model_card == " lllyasviel/sd-controlnet-canny "
and not image_model_card == " Salesforce/blipdiffusion "
and not image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
) :
self . report ( { " INFO " } , " Text prompt in the Generative AI tab is empty! " )
return { " CANCELLED " }
show_system_console ( True )
set_system_console_topmost ( True )
if not seq_editor :
scene . sequence_editor_create ( )
try :
from diffusers import DiffusionPipeline , DPMSolverMultistepScheduler
from diffusers . utils import pt_to_pil
import torch
import requests
from diffusers . utils import load_image
import numpy as np
import PIL
import cv2
from PIL import Image
from . free_lunch_utils import (
register_free_upblock2d ,
register_free_crossattn_upblock2d ,
)
# from compel import Compel
except ModuleNotFoundError :
print ( " Dependencies needs to be installed in the add-on preferences. " )
self . report (
{ " INFO " } ,
" Dependencies needs to be installed in the add-on preferences. " ,
)
return { " CANCELLED " }
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
current_frame = scene . frame_current
type = scene . generatorai_typeselect
input = scene . input_strips
prompt = style_prompt ( scene . generate_movie_prompt ) [ 0 ]
negative_prompt = (
scene . generate_movie_negative_prompt
+ " , "
+ style_prompt ( scene . generate_movie_prompt ) [ 1 ]
+ " , nsfw, nude, nudity, "
)
image_x = scene . generate_movie_x
image_y = scene . generate_movie_y
x = scene . generate_movie_x = closest_divisible_32 ( image_x )
y = scene . generate_movie_y = closest_divisible_32 ( image_y )
duration = scene . generate_movie_frames
image_num_inference_steps = scene . movie_num_inference_steps
image_num_guidance = scene . movie_num_guidance
active_strip = context . scene . sequence_editor . active_strip
do_inpaint = (
input == " input_strips "
and find_strip_by_name ( scene , scene . inpaint_selected_strip )
and type == " image "
and not image_model_card == " lllyasviel/sd-controlnet-canny "
and not image_model_card == " lllyasviel/sd-controlnet-openpose "
and not image_model_card == " lllyasviel/control_v11p_sd15_scribble "
and not image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
and not image_model_card == " Salesforce/blipdiffusion "
)
do_convert = (
( scene . image_path or scene . movie_path )
and not image_model_card == " lllyasviel/sd-controlnet-canny "
and not image_model_card == " lllyasviel/sd-controlnet-openpose "
and not image_model_card == " lllyasviel/control_v11p_sd15_scribble "
and not image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
and not image_model_card == " Salesforce/blipdiffusion "
and not do_inpaint
)
do_refine = (
scene . refine_sd and not do_convert
)
if (
do_inpaint
or do_convert
or image_model_card == " lllyasviel/sd-controlnet-canny "
or image_model_card == " lllyasviel/sd-controlnet-openpose "
or image_model_card == " lllyasviel/control_v11p_sd15_scribble "
or image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
or image_model_card == " Salesforce/blipdiffusion "
) :
if not strips :
self . report ( { " INFO " } , " Select strip(s) for processing. " )
return { " CANCELLED " }
for strip in strips :
if strip . type in { " MOVIE " , " IMAGE " , " TEXT " , " SCENE " } :
break
else :
self . report (
{ " INFO " } ,
" None of the selected strips are movie, image, text or scene types. " ,
)
return { " CANCELLED " }
# LOADING MODELS
# models for inpaint
if do_inpaint :
print ( " Load: Inpaint Model " )
from diffusers import AutoPipelineForInpainting
from diffusers . utils import load_image
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
pipe = AutoPipelineForInpainting . from_pretrained (
" diffusers/stable-diffusion-xl-1.0-inpainting-0.1 " ,
torch_dtype = torch . float16 ,
variant = " fp16 " ,
) . to ( gfx_device )
# Set scheduler
if scene . use_lcm :
from diffusers import LCMScheduler
pipe . scheduler = LCMScheduler . from_config ( pipe . scheduler . config )
# load LCM-LoRA
pipe . load_lora_weights ( " latent-consistency/lcm-lora-sdxl " )
pipe . fuse_lora ( )
pipe . watermark = NoWatermark ( )
if low_vram ( ) :
# torch.cuda.set_per_process_memory_fraction(0.99)
pipe . enable_model_cpu_offload ( )
else :
pipe . to ( gfx_device )
# Conversion img2img/vid2img.
elif (
do_convert
and image_model_card != " warp-ai/wuerstchen "
and image_model_card != " Lykon/dreamshaper-7 "
) :
print ( " Load: img2img/vid2img Model " )
print ( " Conversion Model: " + image_model_card )
if image_model_card == " stabilityai/stable-diffusion-xl-base-1.0 " :
from diffusers import StableDiffusionXLImg2ImgPipeline , AutoencoderKL
vae = AutoencoderKL . from_pretrained (
" madebyollin/sdxl-vae-fp16-fix " , torch_dtype = torch . float16
)
converter = StableDiffusionXLImg2ImgPipeline . from_pretrained (
" stabilityai/stable-diffusion-xl-refiner-1.0 " ,
# text_encoder_2=pipe.text_encoder_2,
vae = vae ,
torch_dtype = torch . float16 ,
variant = " fp16 " ,
)
else :
from diffusers import AutoPipelineForImage2Image
converter = AutoPipelineForImage2Image . from_pretrained (
image_model_card , torch_dtype = torch . float16 , variant = " fp16 "
)
converter . watermark = NoWatermark ( )
if low_vram ( ) :
converter . enable_model_cpu_offload ( )
# refiner.enable_vae_tiling()
# converter.enable_vae_slicing()
else :
converter . to ( gfx_device )
# ControlNet & Illusion
elif (
image_model_card == " lllyasviel/sd-controlnet-canny "
or image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
) :
print ( " Load: ControlNet Model or Illusion " )
from diffusers import (
StableDiffusionControlNetPipeline ,
ControlNetModel ,
UniPCMultistepScheduler ,
)
controlnet = ControlNetModel . from_pretrained (
image_model_card , torch_dtype = torch . float16
)
pipe = StableDiffusionControlNetPipeline . from_pretrained (
" runwayml/stable-diffusion-v1-5 " ,
controlnet = controlnet ,
torch_dtype = torch . float16 ,
safety_checker = None ,
) # safety_checker=None,
if scene . use_lcm :
from diffusers import LCMScheduler
pipe . scheduler = LCMScheduler . from_config ( pipe . scheduler . config )
pipe . load_lora_weights ( " latent-consistency/lcm-lora-sdv1-5 " )
pipe . fuse_lora ( )
scene . movie_num_guidance = 0
else :
pipe . scheduler = UniPCMultistepScheduler . from_config (
pipe . scheduler . config
)
if low_vram ( ) :
pipe . enable_xformers_memory_efficient_attention ( )
pipe . enable_model_cpu_offload ( )
# pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
# Blip
elif image_model_card == " Salesforce/blipdiffusion " :
print ( " Load: Blip Model " )
from diffusers . utils import load_image
import torch
if not find_strip_by_name ( scene , scene . blip_subject_image ) :
from diffusers . pipelines import BlipDiffusionPipeline
pipe = BlipDiffusionPipeline . from_pretrained (
" Salesforce/blipdiffusion " , torch_dtype = torch . float16
) . to ( gfx_device )
else :
from controlnet_aux import CannyDetector
from diffusers . pipelines import BlipDiffusionControlNetPipeline
pipe = BlipDiffusionControlNetPipeline . from_pretrained (
" Salesforce/blipdiffusion-controlnet " , torch_dtype = torch . float16
) . to ( gfx_device )
# OpenPose
elif image_model_card == " lllyasviel/sd-controlnet-openpose " :
print ( " Load: OpenPose Model " )
# NOTE: Is it working on Pose Rig Bones too?
from diffusers import (
StableDiffusionControlNetPipeline ,
ControlNetModel ,
UniPCMultistepScheduler ,
)
from controlnet_aux import OpenposeDetector
# controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16) #safety_checker=None)
# pipe = StableDiffusionControlNetPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16) #safety_checker=None,
openpose = OpenposeDetector . from_pretrained ( " lllyasviel/ControlNet " )
controlnet = ControlNetModel . from_pretrained (
" lllyasviel/sd-controlnet-openpose " , torch_dtype = torch . float16
)
pipe = StableDiffusionControlNetPipeline . from_pretrained (
" runwayml/stable-diffusion-v1-5 " ,
controlnet = controlnet ,
torch_dtype = torch . float16 ,
) # safety_checker=None,
if scene . use_lcm :
from diffusers import LCMScheduler
pipe . scheduler = LCMScheduler . from_config ( pipe . scheduler . config )
pipe . load_lora_weights ( " latent-consistency/lcm-lora-sdv1-5 " )
pipe . fuse_lora ( )
scene . movie_num_guidance = 0
else :
pipe . scheduler = UniPCMultistepScheduler . from_config (
pipe . scheduler . config
)
if low_vram ( ) :
pipe . enable_xformers_memory_efficient_attention ( )
pipe . enable_model_cpu_offload ( )
# pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
# Scribble
elif image_model_card == " lllyasviel/control_v11p_sd15_scribble " :
print ( " Load: Scribble Model " )
from controlnet_aux import PidiNetDetector , HEDdetector
from diffusers import (
ControlNetModel ,
StableDiffusionControlNetPipeline ,
UniPCMultistepScheduler ,
)
processor = HEDdetector . from_pretrained ( " lllyasviel/Annotators " )
checkpoint = " lllyasviel/control_v11p_sd15_scribble "
controlnet = ControlNetModel . from_pretrained (
checkpoint , torch_dtype = torch . float16
)
pipe = StableDiffusionControlNetPipeline . from_pretrained (
" runwayml/stable-diffusion-v1-5 " ,
controlnet = controlnet ,
torch_dtype = torch . float16 ,
)
if scene . use_lcm :
from diffusers import LCMScheduler
pipe . scheduler = LCMScheduler . from_config ( pipe . scheduler . config )
pipe . load_lora_weights ( " latent-consistency/lcm-lora-sdv1-5 " )
pipe . fuse_lora ( )
scene . movie_num_guidance = 0
else :
pipe . scheduler = UniPCMultistepScheduler . from_config (
pipe . scheduler . config
)
if low_vram ( ) :
# torch.cuda.set_per_process_memory_fraction(0.95) # 6 GB VRAM
pipe . enable_model_cpu_offload ( )
# pipe.enable_vae_slicing()
# pipe.enable_forward_chunking(chunk_size=1, dim=1)
else :
pipe . to ( gfx_device )
# Dreamshaper
elif image_model_card == " Lykon/dreamshaper-7 " :
if do_convert :
print (
image_model_card
+ " does not support img2img or img2vid. Ignoring input strip. "
)
from diffusers import DiffusionPipeline
pipe = DiffusionPipeline . from_pretrained (
" Lykon/dreamshaper-7 " , torch_dtype = torch . float16
) # , custom_pipeline="latent_consistency_txt2img"
pipe . to ( torch_device = " cuda " , torch_dtype = torch . float16 )
# Wuerstchen
elif image_model_card == " warp-ai/wuerstchen " :
print ( " Load: Würstchen Model " )
if do_convert :
print (
image_model_card
+ " does not support img2img or img2vid. Ignoring input strip. "
)
from diffusers import AutoPipelineForText2Image
# from diffusers import DiffusionPipeline
from diffusers . pipelines . wuerstchen import DEFAULT_STAGE_C_TIMESTEPS
pipe = AutoPipelineForText2Image . from_pretrained (
" warp-ai/wuerstchen " ,
torch_dtype = torch . float16 ,
)
if low_vram ( ) :
pipe . enable_model_cpu_offload ( )
else :
pipe . to ( gfx_device )
# DeepFloyd
elif image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
print ( " Load: DeepFloyd Model " )
if do_convert :
print (
image_model_card
+ " does not support img2img or img2vid. Ignoring input strip. "
)
from huggingface_hub . commands . user import login
result = login ( token = addon_prefs . hugginface_token )
# stage 1
stage_1 = DiffusionPipeline . from_pretrained (
" DeepFloyd/IF-I-M-v1.0 " , variant = " fp16 " , torch_dtype = torch . float16
)
if low_vram ( ) :
stage_1 . enable_model_cpu_offload ( )
else :
stage_1 . to ( gfx_device )
# stage 2
stage_2 = DiffusionPipeline . from_pretrained (
" DeepFloyd/IF-II-M-v1.0 " ,
text_encoder = None ,
variant = " fp16 " ,
torch_dtype = torch . float16 ,
)
if low_vram ( ) :
stage_2 . enable_model_cpu_offload ( )
else :
stage_2 . to ( gfx_device )
# stage 3
safety_modules = {
" feature_extractor " : stage_1 . feature_extractor ,
" safety_checker " : stage_1 . safety_checker ,
" watermarker " : stage_1 . watermarker ,
}
stage_3 = DiffusionPipeline . from_pretrained (
" stabilityai/stable-diffusion-x4-upscaler " ,
* * safety_modules ,
torch_dtype = torch . float16 ,
)
if low_vram ( ) :
stage_3 . enable_model_cpu_offload ( )
else :
stage_3 . to ( gfx_device )
# Stable diffusion etc.
else :
print ( " Load: " + image_model_card + " Model " )
enabled_items = None
if image_model_card == " stabilityai/stable-diffusion-xl-base-1.0 " :
from diffusers import AutoencoderKL
vae = AutoencoderKL . from_pretrained (
" madebyollin/sdxl-vae-fp16-fix " , torch_dtype = torch . float16
)
pipe = DiffusionPipeline . from_pretrained (
image_model_card ,
vae = vae ,
torch_dtype = torch . float16 ,
variant = " fp16 " ,
)
elif image_model_card == " runwayml/stable-diffusion-v1-5 " :
from diffusers import StableDiffusionPipeline
pipe = StableDiffusionPipeline . from_pretrained (
" runwayml/stable-diffusion-v1-5 " ,
torch_dtype = torch . float16 , # vae=vae,
)
else :
from diffusers import AutoPipelineForText2Image
pipe = AutoPipelineForText2Image . from_pretrained (
image_model_card ,
torch_dtype = torch . float16 ,
variant = " fp16 " ,
)
if scene . use_lcm :
print ( " Use LCM: True " )
from diffusers import LCMScheduler
if image_model_card == " stabilityai/stable-diffusion-xl-base-1.0 " :
scene . movie_num_guidance = 0
pipe . load_lora_weights ( " latent-consistency/lcm-lora-sdxl " )
pipe . fuse_lora ( )
elif image_model_card == " segmind/SSD-1B " :
scene . movie_num_guidance = 0
pipe . load_lora_weights ( " latent-consistency/lcm-lora-ssd-1b " )
pipe . fuse_lora ( )
elif image_model_card == " latent-consistency/lcm-lora-sdv1-5 " :
scene . movie_num_guidance = 0
pipe . load_lora_weights ( " latent-consistency/lcm-lora-sdv1-5 " )
pipe . fuse_lora ( )
pipe . scheduler = LCMScheduler . from_config ( pipe . scheduler . config )
else :
print ( " Use LCM: False " )
pipe . scheduler = DPMSolverMultistepScheduler . from_config (
pipe . scheduler . config
)
pipe . watermark = NoWatermark ( )
if low_vram ( ) :
# torch.cuda.set_per_process_memory_fraction(0.95) # 6 GB VRAM
pipe . enable_model_cpu_offload ( )
#pipe.enable_vae_slicing()
else :
pipe . to ( gfx_device )
if scene . use_freeU and pipe : # Free Lunch
# -------- freeu block registration
print ( " Process: FreeU " )
register_free_upblock2d ( pipe , b1 = 1.1 , b2 = 1.2 , s1 = 0.6 , s2 = 0.4 )
register_free_crossattn_upblock2d ( pipe , b1 = 1.1 , b2 = 1.2 , s1 = 0.6 , s2 = 0.4 )
# -------- freeu block registration
# LoRA
if (
image_model_card == " stabilityai/stable-diffusion-xl-base-1.0 "
or image_model_card == " runwayml/stable-diffusion-v1-5 "
or image_model_card == " stabilityai/sdxl-turbo "
) :
scene = context . scene
lora_files = scene . lora_files
enabled_names = [ ]
enabled_weights = [ ]
# Check if there are any enabled items before loading
enabled_items = [ item for item in lora_files if item . enabled ]
if enabled_items :
for item in enabled_items :
enabled_names . append (
( clean_filename ( item . name ) ) . replace ( " . " , " " )
)
enabled_weights . append ( item . weight_value )
pipe . load_lora_weights (
bpy . path . abspath ( scene . lora_folder ) ,
weight_name = item . name + " .safetensors " ,
adapter_name = ( ( clean_filename ( item . name ) ) . replace ( " . " , " " ) ) ,
)
pipe . set_adapters ( enabled_names , adapter_weights = enabled_weights )
print ( " Load LoRAs: " + " " . join ( enabled_names ) )
# load refiner model if chosen.
if do_refine :
print (
" Load Refine Model: " + " stabilityai/stable-diffusion-xl-refiner-1.0 "
)
from diffusers import StableDiffusionXLImg2ImgPipeline , AutoencoderKL
vae = AutoencoderKL . from_pretrained (
" madebyollin/sdxl-vae-fp16-fix " , torch_dtype = torch . float16
)
refiner = StableDiffusionXLImg2ImgPipeline . from_pretrained (
" stabilityai/stable-diffusion-xl-refiner-1.0 " ,
vae = vae ,
torch_dtype = torch . float16 ,
variant = " fp16 " ,
)
refiner . watermark = NoWatermark ( )
if low_vram ( ) :
refiner . enable_model_cpu_offload ( )
#refiner.enable_vae_tiling()
#refiner.enable_vae_slicing()
else :
refiner . to ( gfx_device )
# # Allow longer prompts.
# if image_model_card == "runwayml/stable-diffusion-v1-5":
# if pipe:
# compel = Compel(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder)
# if refiner:
# compel = Compel(tokenizer=refiner.tokenizer, text_encoder=refiner.text_encoder)
# if converter:
# compel = Compel(tokenizer=converter.tokenizer, text_encoder=converter.text_encoder)
# prompt_embed = compel.build_conditioning_tensor(prompt)
# Main Generate Loop:
for i in range ( scene . movie_num_batch ) :
# Find free space for the strip in the timeline.
if i > 0 :
empty_channel = scene . sequence_editor . active_strip . channel
start_frame = (
scene . sequence_editor . active_strip . frame_final_start
+ scene . sequence_editor . active_strip . frame_final_duration
)
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
else :
empty_channel = find_first_empty_channel (
scene . frame_current ,
( scene . movie_num_batch * duration ) + scene . frame_current ,
)
start_frame = scene . frame_current
# Generate seed.
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( - 2147483647 , 2147483647 )
)
print ( " Seed: " + str ( seed ) )
context . scene . movie_num_seed = seed
# Use cuda if possible.
if torch . cuda . is_available ( ) :
generator = (
torch . Generator ( " cuda " ) . manual_seed ( seed ) if seed != 0 else None
)
else :
if seed != 0 :
generator = torch . Generator ( )
generator . manual_seed ( seed )
else :
generator = None
# DeepFloyd process:
if image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
prompt_embeds , negative_embeds = stage_1 . encode_prompt (
prompt , negative_prompt
)
# stage 1
image = stage_1 (
prompt_embeds = prompt_embeds ,
negative_prompt_embeds = negative_embeds ,
generator = generator ,
output_type = " pt " ,
) . images
pt_to_pil ( image ) [ 0 ] . save ( " ./if_stage_I.png " )
# stage 2
image = stage_2 (
image = image ,
prompt_embeds = prompt_embeds ,
negative_prompt_embeds = negative_embeds ,
generator = generator ,
output_type = " pt " ,
) . images
pt_to_pil ( image ) [ 0 ] . save ( " ./if_stage_II.png " )
# stage 3
image = stage_3 (
prompt = prompt , image = image , noise_level = 100 , generator = generator
) . images
# image[0].save("./if_stage_III.png")
image = image [ 0 ]
elif image_model_card == " warp-ai/wuerstchen " :
scene . generate_movie_y = y = closest_divisible_128 ( y )
scene . generate_movie_x = x = closest_divisible_128 ( x )
print ( " Generate: Image with Würstchen " )
image = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
decoder_guidance_scale = 0.0 ,
# image_embeddings=None,
prior_guidance_scale = image_num_guidance ,
prior_timesteps = DEFAULT_STAGE_C_TIMESTEPS ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# ControlNet
elif (
image_model_card == " lllyasviel/sd-controlnet-canny "
or image_model_card == " monster-labs/control_v1p_sd15_qrcode_monster "
) :
print ( " Process: ControlNet " )
init_image = None
if scene . image_path :
init_image = load_first_frame ( scene . image_path )
if scene . movie_path :
init_image = load_first_frame ( scene . movie_path )
if not init_image :
print ( " Loading strip failed! " )
return { " CANCELLED " }
init_image = init_image . resize ( ( x , y ) )
if image_model_card == " lllyasviel/sd-controlnet-canny " :
image = np . array ( init_image )
low_threshold = 100
high_threshold = 200
image = cv2 . Canny ( image , low_threshold , high_threshold )
image = image [ : , : , None ]
canny_image = np . concatenate ( [ image , image , image ] , axis = 2 )
canny_image = Image . fromarray ( canny_image )
# canny_image = np.array(canny_image)
else :
canny_image = init_image
image = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
image = canny_image ,
num_inference_steps = image_num_inference_steps , # Should be around 50
# guidance_scale=clamp_value(
# image_num_guidance, 3, 5
# ), # Should be between 3 and 5.
# # guess_mode=True, #NOTE: Maybe the individual methods should be selectable instead?
# height=y,
# width=x,
# generator=generator,
) . images [ 0 ]
elif image_model_card == " Lykon/dreamshaper-7 " :
image = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
lcm_origin_steps = 50 ,
height = y ,
width = x ,
generator = generator ,
output_type = " pil " ,
) . images [ 0 ]
# OpenPose
elif image_model_card == " lllyasviel/sd-controlnet-openpose " :
print ( " Process: OpenPose " )
init_image = None
if scene . image_path :
init_image = load_first_frame ( scene . image_path )
if scene . movie_path :
init_image = load_first_frame ( scene . movie_path )
if not init_image :
print ( " Loading strip failed! " )
return { " CANCELLED " }
image = init_image . resize ( ( x , y ) )
if not scene . openpose_use_bones :
image = np . array ( image )
image = openpose ( image )
image = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
image = image ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# Scribble
elif image_model_card == " lllyasviel/control_v11p_sd15_scribble " :
print ( " Process: Scribble " )
init_image = None
if scene . image_path :
init_image = load_first_frame ( scene . image_path )
if scene . movie_path :
init_image = load_first_frame ( scene . movie_path )
if not init_image :
print ( " Loading strip failed! " )
return { " CANCELLED " }
image = init_image . resize ( ( x , y ) )
if scene . use_scribble_image :
image = np . array ( image )
image = cv2 . cvtColor ( image , cv2 . COLOR_BGR2GRAY )
image = cv2 . bitwise_not ( image )
image = processor ( image , scribble = False )
else :
image = np . array ( image )
image = processor ( image , scribble = True )
image = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
image = image ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# Blip
elif image_model_card == " Salesforce/blipdiffusion " :
print ( " Process: Subject Driven " )
text_prompt_input = prompt
style_subject = str ( scene . blip_cond_subject )
tgt_subject = str ( scene . blip_tgt_subject )
init_image = None
if scene . image_path :
init_image = load_first_frame ( scene . image_path )
if scene . movie_path :
init_image = load_first_frame ( scene . movie_path )
if not init_image :
print ( " Loading strip failed! " )
return { " CANCELLED " }
init_image = init_image . resize ( ( x , y ) )
style_image = init_image
subject_strip = find_strip_by_name ( scene , scene . blip_subject_image )
if subject_strip :
if (
subject_strip . type == " MASK "
or subject_strip . type == " COLOR "
or subject_strip . type == " SCENE "
or subject_strip . type == " META "
) :
subject_strip = get_render_strip ( self , context , subject_strip )
subject_path = get_strip_path ( subject_strip )
cldm_cond_image = load_first_frame ( subject_path )
canny = CannyDetector ( )
cldm_cond_image = canny ( cldm_cond_image , 30 , 70 , output_type = " pil " )
if cldm_cond_image :
cldm_cond_image = cldm_cond_image . resize ( ( x , y ) )
image = pipe (
text_prompt_input ,
style_image ,
cldm_cond_image ,
style_subject ,
tgt_subject ,
guidance_scale = image_num_guidance ,
num_inference_steps = image_num_inference_steps ,
neg_prompt = negative_prompt ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
else :
print ( " Subject strip loading failed! " )
subject_strip = " "
if not subject_strip :
image = pipe (
text_prompt_input ,
style_image ,
style_subject ,
tgt_subject ,
guidance_scale = image_num_guidance ,
num_inference_steps = image_num_inference_steps ,
neg_prompt = negative_prompt ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# Inpaint
elif do_inpaint :
print ( " Process: Inpaint " )
mask_strip = find_strip_by_name ( scene , scene . inpaint_selected_strip )
if not mask_strip :
print ( " Selected mask not found! " )
return { " CANCELLED " }
if (
mask_strip . type == " MASK "
or mask_strip . type == " COLOR "
or mask_strip . type == " SCENE "
or mask_strip . type == " META "
) :
mask_strip = get_render_strip ( self , context , mask_strip )
mask_path = get_strip_path ( mask_strip )
mask_image = load_first_frame ( mask_path )
if not mask_image :
print ( " Loading mask failed! " )
return
mask_image = mask_image . resize ( ( x , y ) )
if scene . image_path :
init_image = load_first_frame ( scene . image_path )
if scene . movie_path :
init_image = load_first_frame ( scene . movie_path )
if not init_image :
print ( " Loading strip failed! " )
return { " CANCELLED " }
init_image = init_image . resize ( ( x , y ) )
image = pipe (
prompt = prompt ,
negative_prompt = negative_prompt ,
image = init_image ,
mask_image = mask_image ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# Limit inpaint to maske area:
# Convert mask to grayscale NumPy array
mask_image_arr = np . array ( mask_image . convert ( " L " ) )
# Add a channel dimension to the end of the grayscale mask
mask_image_arr = mask_image_arr [ : , : , None ]
mask_image_arr = mask_image_arr . astype ( np . float32 ) / 255.0
mask_image_arr [ mask_image_arr < 0.5 ] = 0
mask_image_arr [ mask_image_arr > = 0.5 ] = 1
# Take the masked pixels from the repainted image and the unmasked pixels from the initial image
unmasked_unchanged_image_arr = (
1 - mask_image_arr
) * init_image + mask_image_arr * image
image = PIL . Image . fromarray (
unmasked_unchanged_image_arr . astype ( " uint8 " )
)
delete_strip ( mask_strip )
# Img2img
elif do_convert :
if scene . movie_path :
print ( " Process: Video to Image " )
init_image = load_first_frame ( scene . movie_path )
init_image = init_image . resize ( ( x , y ) )
elif scene . image_path :
print ( " Process: Image to Image " )
init_image = load_first_frame ( scene . image_path )
init_image = init_image . resize ( ( x , y ) )
# init_image = load_image(scene.image_path).convert("RGB")
if image_model_card == " stabilityai/sdxl-turbo " or image_model_card == " stabilityai/sd-turbo " :
image = converter (
prompt = prompt ,
image = init_image ,
strength = 1.00 - scene . image_power ,
# negative_prompt=negative_prompt,
num_inference_steps = image_num_inference_steps ,
guidance_scale = 0.0 ,
generator = generator ,
) . images [ 0 ]
else :
image = converter (
prompt = prompt ,
image = init_image ,
strength = 1.00 - scene . image_power ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
generator = generator ,
) . images [ 0 ]
# Generate Stable Diffusion etc.
else :
print ( " Generate: Image " )
if image_model_card == " stabilityai/sdxl-turbo " :
# LoRA.
if enabled_items :
image = pipe (
# prompt_embeds=prompt, # for compel - long prompts
prompt ,
# negative_prompt=negative_prompt,
num_inference_steps = image_num_inference_steps ,
guidance_scale = 0.0 ,
height = y ,
width = x ,
cross_attention_kwargs = { " scale " : 1.0 } ,
generator = generator ,
) . images [ 0 ]
# No LoRA.
else :
image = pipe (
# prompt_embeds=prompt, # for compel - long prompts
prompt ,
# negative_prompt=negative_prompt,
num_inference_steps = image_num_inference_steps ,
guidance_scale = 0.0 ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
else :
# LoRA.
if enabled_items :
image = pipe (
# prompt_embeds=prompt, # for compel - long prompts
prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
height = y ,
width = x ,
cross_attention_kwargs = { " scale " : 1.0 } ,
generator = generator ,
) . images [ 0 ]
# No LoRA.
else :
image = pipe (
# prompt_embeds=prompt, # for compel - long prompts
prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# Add refiner
if do_refine :
print ( " Refine: Image " )
image = refiner (
prompt = prompt ,
image = image ,
strength = max ( 1.00 - scene . image_power , 0.1 ) ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = max ( image_num_guidance , 1.1 ) ,
generator = generator ,
) . images [ 0 ]
# Move to folder
filename = clean_filename (
str ( seed ) + " _ " + context . scene . generate_movie_prompt
)
out_path = solve_path ( filename + " .png " )
image . save ( out_path )
# Add strip
if os . path . isfile ( out_path ) :
strip = scene . sequence_editor . sequences . new_image (
name = str ( seed ) + " _ " + context . scene . generate_movie_prompt ,
frame_start = start_frame ,
filepath = out_path ,
channel = empty_channel ,
fit_method = " FIT " ,
)
strip . frame_final_duration = scene . generate_movie_frames
strip . transform . filter = " SUBSAMPLING_3x3 "
scene . sequence_editor . active_strip = strip
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
strip . use_proxy = True
# bpy.ops.sequencer.rebuild_proxy()
else :
print ( " No resulting file found. " )
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
for window in bpy . context . window_manager . windows :
screen = window . screen
for area in screen . areas :
if area . type == " SEQUENCE_EDITOR " :
from bpy import context
with context . temp_override ( window = window , area = area ) :
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip. Remove this if Blender crashes: https://docs.blender.org/api/current/info_gotcha.html#can-i-redraw-during-script-execution
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
break
scene . movie_num_guidance = guidance
bpy . ops . renderreminder . play_notification ( )
scene . frame_current = current_frame
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
return { " FINISHED " }
def clean_string ( input_string ) :
# Words to be removed
words_to_remove = [ " araffe " , " arafed " , " there is " , " there are " ]
for word in words_to_remove :
input_string = input_string . replace ( word , " " )
input_string = input_string . strip ( )
# Capitalize the first letter
input_string = input_string [ : 1 ] . capitalize ( ) + input_string [ 1 : ]
# Add a full stop at the end
input_string + = " . "
return input_string
class SEQUENCER_OT_generate_text ( Operator ) :
""" Generate Text """
bl_idname = " sequencer.generate_text "
bl_label = " Prompt "
bl_description = " Generate texts from strips "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
seq_editor = scene . sequence_editor
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
guidance = scene . movie_num_guidance
current_frame = scene . frame_current
prompt = style_prompt ( scene . generate_movie_prompt ) [ 0 ]
x = scene . generate_movie_x = closest_divisible_32 ( scene . generate_movie_x )
y = scene . generate_movie_y = closest_divisible_32 ( scene . generate_movie_y )
duration = scene . generate_movie_frames
render = bpy . context . scene . render
fps = render . fps / render . fps_base
show_system_console ( True )
set_system_console_topmost ( True )
if not seq_editor :
scene . sequence_editor_create ( )
active_strip = context . scene . sequence_editor . active_strip
try :
import torch
from PIL import Image
from transformers import BlipProcessor , BlipForConditionalGeneration
except ModuleNotFoundError :
print ( " Dependencies need to be installed in the add-on preferences. " )
self . report (
{ " INFO " } ,
" Dependencies need to be installed in the add-on preferences. " ,
)
return { " CANCELLED " }
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
processor = BlipProcessor . from_pretrained (
" Salesforce/blip-image-captioning-large "
)
model = BlipForConditionalGeneration . from_pretrained (
" Salesforce/blip-image-captioning-large " , torch_dtype = torch . float16
) . to ( gfx_device )
init_image = (
load_first_frame ( scene . movie_path )
if scene . movie_path
else load_first_frame ( scene . image_path )
)
init_image = init_image . resize ( ( x , y ) )
text = " "
inputs = processor ( init_image , text , return_tensors = " pt " ) . to (
gfx_device , torch . float16
)
out = model . generate ( * * inputs , max_new_tokens = 256 )
text = processor . decode ( out [ 0 ] , skip_special_tokens = True )
text = clean_string ( text )
print ( " Generated text: " + text )
# Find free space for the strip in the timeline.
if (
active_strip . frame_final_start
< = current_frame
< = ( active_strip . frame_final_start + active_strip . frame_final_duration )
) :
empty_channel = find_first_empty_channel (
scene . frame_current ,
( scene . sequence_editor . active_strip . frame_final_duration )
+ scene . frame_current ,
)
start_frame = scene . frame_current
else :
empty_channel = find_first_empty_channel (
scene . sequence_editor . active_strip . frame_final_start ,
scene . sequence_editor . active_strip . frame_final_end ,
)
start_frame = scene . sequence_editor . active_strip . frame_final_start
scene . frame_current = scene . sequence_editor . active_strip . frame_final_start
# Add strip
if text :
print ( str ( start_frame ) )
strip = scene . sequence_editor . sequences . new_effect (
name = text ,
type = " TEXT " ,
frame_start = start_frame ,
frame_end = int ( start_frame + ( ( len ( text ) / 12 ) * fps ) ) ,
channel = empty_channel ,
)
strip . text = text
strip . wrap_width = 0.68
strip . font_size = 44
strip . location [ 0 ] = 0.5
strip . location [ 1 ] = 0.2
strip . align_x = " CENTER "
strip . align_y = " TOP "
strip . use_shadow = True
strip . use_box = True
scene . sequence_editor . active_strip = strip
for window in bpy . context . window_manager . windows :
screen = window . screen
for area in screen . areas :
if area . type == " SEQUENCE_EDITOR " :
from bpy import context
with context . temp_override ( window = window , area = area ) :
if (
active_strip . frame_final_start
< = scene . frame_current
< = (
active_strip . frame_final_start
+ active_strip . frame_final_duration
)
) :
pass
else :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip.
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
break
scene . movie_num_guidance = guidance
bpy . ops . renderreminder . play_notification ( )
scene . frame_current = current_frame
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
return { " FINISHED " }
class SEQUENCER_OT_strip_to_generatorAI ( Operator ) :
""" Convert selected text strips to Generative AI """
bl_idname = " sequencer.text_to_generator "
bl_label = " Generative AI "
bl_options = { " INTERNAL " }
bl_description = " Adds selected strips as inputs to the Generative AI process "
@classmethod
def poll ( cls , context ) :
return context . scene and context . scene . sequence_editor
def execute ( self , context ) :
bpy . types . Scene . movie_path = " "
bpy . types . Scene . image_path = " "
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
play_sound = addon_prefs . playsound
addon_prefs . playsound = False
scene = context . scene
sequencer = bpy . ops . sequencer
sequences = bpy . context . sequences
strips = context . selected_sequences
active_strip = context . scene . sequence_editor . active_strip
prompt = scene . generate_movie_prompt
negative_prompt = scene . generate_movie_negative_prompt
current_frame = scene . frame_current
type = scene . generatorai_typeselect
seed = scene . movie_num_seed
use_random = scene . movie_use_random
use_strip_data = addon_prefs . use_strip_data
temp_strip = None
if not strips :
self . report ( { " INFO " } , " Select strip(s) for processing. " )
return { " CANCELLED " }
else :
print ( " \n Strip input processing started (Ctrl+C to Cancel). " )
for strip in strips :
if strip . type in { " MOVIE " , " IMAGE " , " TEXT " , " SCENE " } :
break
else :
self . report (
{ " INFO " } ,
" None of the selected strips are movie, image, text or scene types. " ,
)
return { " CANCELLED " }
if type == " text " :
for strip in strips :
if strip . type in { " MOVIE " , " IMAGE " } :
print ( " Process: Image Captioning " )
break
else :
self . report (
{ " INFO " } ,
" None of the selected strips are movie or image. " ,
)
return { " CANCELLED " }
if use_strip_data :
print ( " Use file seed and prompt: Yes " )
else :
print ( " Use file seed and prompt: No " )
import torch
import scipy
total_vram = 0
for i in range ( torch . cuda . device_count ( ) ) :
properties = torch . cuda . get_device_properties ( i )
total_vram + = properties . total_memory
print ( " Total VRAM: " + str ( total_vram ) )
print ( " Total GPU Cards: " + str ( torch . cuda . device_count ( ) ) )
for count , strip in enumerate ( strips ) :
for dsel_strip in bpy . context . scene . sequence_editor . sequences :
dsel_strip . select = False
strip . select = True
# render intermediate mp4 file
if strip . type == " SCENE " or strip . type == " MOVIE " :
# Make the current frame overlapped frame, the temp strip.
if type == " image " or type == " text " :
trim_frame = find_overlapping_frame ( strip , current_frame )
if trim_frame and len ( strips ) == 1 :
bpy . ops . sequencer . copy ( )
bpy . ops . sequencer . paste ( )
intermediate_strip = bpy . context . selected_sequences [ 0 ]
intermediate_strip . frame_start = strip . frame_start
intermediate_strip . frame_offset_start = int ( trim_frame )
intermediate_strip . frame_final_duration = 1
temp_strip = strip = get_render_strip (
self , context , intermediate_strip
)
if intermediate_strip is not None :
delete_strip ( intermediate_strip )
elif type == " text " :
bpy . ops . sequencer . copy ( )
bpy . ops . sequencer . paste ( keep_offset = True )
intermediate_strip = bpy . context . selected_sequences [ 0 ]
intermediate_strip . frame_start = strip . frame_start
# intermediate_strip.frame_offset_start = int(trim_frame)
intermediate_strip . frame_final_duration = 1
temp_strip = strip = get_render_strip (
self , context , intermediate_strip
)
if intermediate_strip is not None :
delete_strip ( intermediate_strip )
else :
temp_strip = strip = get_render_strip ( self , context , strip )
else :
temp_strip = strip = get_render_strip ( self , context , strip )
if strip . type == " TEXT " :
if strip . text :
print ( " \n " + str ( count + 1 ) + " / " + str ( len ( strips ) ) )
print ( " Prompt: " + strip . text + " , " + prompt )
print ( " Negative Prompt: " + negative_prompt )
scene . generate_movie_prompt = strip . text + " , " + prompt
scene . frame_current = strip . frame_final_start
if type == " movie " :
sequencer . generate_movie ( )
if type == " audio " :
sequencer . generate_audio ( )
if type == " image " :
sequencer . generate_image ( )
context . scene . generate_movie_prompt = prompt
scene . generate_movie_negative_prompt = negative_prompt
context . scene . movie_use_random = use_random
context . scene . movie_num_seed = seed
scene . generate_movie_prompt = prompt
scene . generate_movie_negative_prompt = negative_prompt
if use_strip_data :
scene . movie_use_random = use_random
scene . movie_num_seed = seed
if strip . type == " IMAGE " :
strip_dirname = os . path . dirname ( strip . directory )
image_path = bpy . path . abspath (
os . path . join ( strip_dirname , strip . elements [ 0 ] . filename )
)
bpy . types . Scene . image_path = image_path
if strip . name :
strip_prompt = os . path . splitext ( strip . name ) [ 0 ]
seed_nr = extract_numbers ( str ( strip_prompt ) )
if seed_nr :
file_seed = int ( seed_nr )
if file_seed and use_strip_data :
strip_prompt = strip_prompt . replace (
str ( file_seed ) + " _ " , " "
)
context . scene . movie_use_random = False
context . scene . movie_num_seed = file_seed
if use_strip_data :
styled_prompt = style_prompt ( strip_prompt + " , " + prompt ) [ 0 ]
styled_negative_prompt = style_prompt (
strip_prompt + " , " + prompt
) [ 1 ]
else :
styled_prompt = style_prompt ( prompt ) [ 0 ]
styled_negative_prompt = style_prompt ( prompt ) [ 1 ]
print ( " \n " + str ( count + 1 ) + " / " + str ( len ( strips ) ) )
if type != " text " :
print ( " Prompt: " + styled_prompt )
print ( " Negative Prompt: " + styled_negative_prompt )
scene . generate_movie_prompt = styled_prompt
scene . generate_movie_negative_prompt = styled_negative_prompt
scene . frame_current = strip . frame_final_start
context . scene . sequence_editor . active_strip = strip
if type == " movie " :
sequencer . generate_movie ( )
if type == " audio " :
sequencer . generate_audio ( )
if type == " image " :
sequencer . generate_image ( )
if type == " text " :
sequencer . generate_text ( )
scene . generate_movie_prompt = prompt
scene . generate_movie_negative_prompt = negative_prompt
if use_strip_data :
scene . movie_use_random = use_random
scene . movie_num_seed = seed
bpy . types . Scene . image_path = " "
if strip . type == " MOVIE " :
movie_path = bpy . path . abspath ( strip . filepath )
bpy . types . Scene . movie_path = movie_path
if strip . name :
strip_prompt = os . path . splitext ( strip . name ) [ 0 ]
seed_nr = extract_numbers ( str ( strip_prompt ) )
if seed_nr :
file_seed = int ( seed_nr )
if file_seed and use_strip_data :
strip_prompt = strip_prompt . replace (
str ( file_seed ) + " _ " , " "
)
context . scene . movie_use_random = False
context . scene . movie_num_seed = file_seed
if use_strip_data :
styled_prompt = style_prompt ( strip_prompt + " , " + prompt ) [ 0 ]
styled_negative_prompt = style_prompt (
strip_prompt + " , " + prompt
) [ 1 ]
else :
styled_prompt = style_prompt ( prompt ) [ 0 ]
styled_negative_prompt = style_prompt ( prompt ) [ 1 ]
print ( " \n " + str ( count + 1 ) + " / " + str ( len ( strips ) ) )
if type != " text " :
print ( " Prompt: " + styled_prompt )
print ( " Negative Prompt: " + styled_negative_prompt )
scene . generate_movie_prompt = styled_prompt
scene . generate_movie_negative_prompt = styled_negative_prompt
scene . frame_current = strip . frame_final_start
context . scene . sequence_editor . active_strip = strip
if type == " movie " :
sequencer . generate_movie ( )
if type == " audio " :
sequencer . generate_audio ( )
if type == " image " :
sequencer . generate_image ( )
if type == " text " :
sequencer . generate_text ( )
scene . generate_movie_prompt = prompt
scene . generate_movie_negative_prompt = negative_prompt
if use_strip_data :
scene . movie_use_random = use_random
scene . movie_num_seed = seed
if temp_strip is not None :
delete_strip ( temp_strip )
bpy . types . Scene . movie_path = " "
scene . generate_movie_prompt = prompt
scene . generate_movie_negative_prompt = negative_prompt
context . scene . movie_use_random = use_random
context . scene . movie_num_seed = seed
scene . frame_current = current_frame
scene . generate_movie_prompt = prompt
scene . generate_movie_negative_prompt = negative_prompt
context . scene . movie_use_random = use_random
context . scene . movie_num_seed = seed
context . scene . sequence_editor . active_strip = active_strip
addon_prefs . playsound = play_sound
bpy . ops . renderreminder . play_notification ( )
print ( " Processing finished. " )
return { " FINISHED " }
classes = (
GeneratorAddonPreferences ,
SEQUENCER_OT_generate_movie ,
SEQUENCER_OT_generate_audio ,
SEQUENCER_OT_generate_image ,
SEQUENCER_OT_generate_text ,
SEQUENCER_PT_pallaidium_panel ,
GENERATOR_OT_sound_notification ,
SEQUENCER_OT_strip_to_generatorAI ,
LORABrowserFileItem ,
LORA_OT_RefreshFiles ,
LORABROWSER_UL_files ,
GENERATOR_OT_install ,
GENERATOR_OT_uninstall ,
)
def register ( ) :
bpy . types . Scene . generate_movie_prompt = bpy . props . StringProperty (
name = " generate_movie_prompt " ,
default = " " ,
options = { " TEXTEDIT_UPDATE " } ,
)
bpy . types . Scene . generate_movie_negative_prompt = bpy . props . StringProperty (
name = " generate_movie_negative_prompt " ,
default = " " ,
options = { " TEXTEDIT_UPDATE " } ,
)
bpy . types . Scene . generate_audio_prompt = bpy . props . StringProperty (
name = " generate_audio_prompt " , default = " "
)
bpy . types . Scene . generate_movie_x = bpy . props . IntProperty (
name = " generate_movie_x " ,
default = 1024 ,
step = 64 ,
min = 256 ,
max = 1536 ,
)
bpy . types . Scene . generate_movie_y = bpy . props . IntProperty (
name = " generate_movie_y " ,
default = 512 ,
step = 64 ,
min = 256 ,
max = 1536 ,
)
# The number of frames to be generated.
bpy . types . Scene . generate_movie_frames = bpy . props . IntProperty (
name = " generate_movie_frames " ,
default = 6 ,
min = 1 ,
max = 125 ,
)
# The number of denoising steps. More denoising steps usually lead to a higher quality audio at the expense of slower inference.
bpy . types . Scene . movie_num_inference_steps = bpy . props . IntProperty (
name = " movie_num_inference_steps " ,
default = 25 ,
min = 1 ,
max = 100 ,
)
# The number of videos to generate.
bpy . types . Scene . movie_num_batch = bpy . props . IntProperty (
name = " movie_num_batch " ,
default = 1 ,
min = 1 ,
max = 100 ,
)
# The seed number.
bpy . types . Scene . movie_num_seed = bpy . props . IntProperty (
name = " movie_num_seed " ,
default = 1 ,
min = - 2147483647 ,
max = 2147483647 ,
)
# The seed number.
bpy . types . Scene . movie_use_random = bpy . props . BoolProperty (
name = " movie_use_random " ,
default = 1 ,
)
# The guidance number.
bpy . types . Scene . movie_num_guidance = bpy . props . FloatProperty (
name = " movie_num_guidance " ,
default = 4.0 ,
min = 0 ,
max = 100 ,
)
# The frame audio duration.
bpy . types . Scene . audio_length_in_f = bpy . props . IntProperty (
name = " audio_length_in_f " ,
default = 80 ,
min = 1 ,
max = 10000 ,
)
bpy . types . Scene . generatorai_typeselect = bpy . props . EnumProperty (
name = " Sound " ,
items = [
( " movie " , " Video " , " Generate Video " ) ,
( " image " , " Image " , " Generate Image " ) ,
( " audio " , " Audio " , " Generate Audio " ) ,
( " text " , " Text " , " Generate Text " ) ,
] ,
default = " image " ,
update = output_strips_updated ,
)
bpy . types . Scene . speakers = bpy . props . EnumProperty (
name = " Speakers " ,
items = [
( " speaker_0 " , " Speaker 0 " , " " ) ,
( " speaker_1 " , " Speaker 1 " , " " ) ,
( " speaker_2 " , " Speaker 2 " , " " ) ,
( " speaker_3 " , " Speaker 3 " , " " ) ,
( " speaker_4 " , " Speaker 4 " , " " ) ,
( " speaker_5 " , " Speaker 5 " , " " ) ,
( " speaker_6 " , " Speaker 6 " , " " ) ,
( " speaker_7 " , " Speaker 7 " , " " ) ,
( " speaker_8 " , " Speaker 8 " , " " ) ,
( " speaker_9 " , " Speaker 9 " , " " ) ,
] ,
default = " speaker_3 " ,
)
bpy . types . Scene . languages = bpy . props . EnumProperty (
name = " Languages " ,
items = [
( " en " , " English " , " " ) ,
( " de " , " German " , " " ) ,
( " es " , " Spanish " , " " ) ,
( " fr " , " French " , " " ) ,
( " hi " , " Hindi " , " " ) ,
( " it " , " Italian " , " " ) ,
( " ja " , " Japanese " , " " ) ,
( " ko " , " Korean " , " " ) ,
( " pl " , " Polish " , " " ) ,
( " pt " , " Portuguese " , " " ) ,
( " ru " , " Russian " , " " ) ,
( " tr " , " Turkish " , " " ) ,
( " zh " , " Chinese, simplified " , " " ) ,
] ,
default = " en " ,
)
# Inpaint
bpy . types . Scene . inpaint_selected_strip = bpy . props . StringProperty (
name = " inpaint_selected_strip " , default = " "
)
# Upscale
bpy . types . Scene . video_to_video = bpy . props . BoolProperty (
name = " video_to_video " ,
default = 0 ,
)
# Refine SD
bpy . types . Scene . refine_sd = bpy . props . BoolProperty (
name = " refine_sd " ,
default = 1 ,
)
# movie path
bpy . types . Scene . movie_path = bpy . props . StringProperty ( name = " movie_path " , default = " " )
bpy . types . Scene . movie_path = " "
# image path
bpy . types . Scene . image_path = bpy . props . StringProperty ( name = " image_path " , default = " " )
bpy . types . Scene . image_path = " "
bpy . types . Scene . input_strips = bpy . props . EnumProperty (
items = [
( " generate " , " No Input " , " No Input " ) ,
( " input_strips " , " Strips " , " Selected Strips " ) ,
] ,
default = " generate " ,
update = input_strips_updated ,
)
bpy . types . Scene . image_power = bpy . props . FloatProperty (
name = " image_power " ,
default = 0.50 ,
min = 0.05 ,
max = 0.82 ,
)
styles_array = load_styles (
os . path . dirname ( os . path . abspath ( __file__ ) ) + " /styles.json "
)
if styles_array :
bpy . types . Scene . generatorai_styles = bpy . props . EnumProperty (
name = " Generator AI Styles " ,
items = [ ( " no_style " , " No Style " , " No Style " ) ] + styles_array ,
default = " no_style " ,
)
bpy . types . Scene . openpose_use_bones = bpy . props . BoolProperty (
name = " openpose_use_bones " ,
default = 0 ,
)
bpy . types . Scene . use_scribble_image = bpy . props . BoolProperty (
name = " use_scribble_image " ,
default = 0 ,
)
# Blip
bpy . types . Scene . blip_cond_subject = bpy . props . StringProperty (
name = " blip_cond_subject " ,
default = " " ,
)
bpy . types . Scene . blip_tgt_subject = bpy . props . StringProperty (
name = " blip_tgt_subject " ,
default = " " ,
)
bpy . types . Scene . blip_subject_image = bpy . props . StringProperty (
name = " blip_subject_image " , default = " "
)
bpy . types . Scene . use_freeU = bpy . props . BoolProperty (
name = " use_freeU " ,
default = 0 ,
)
bpy . types . Scene . use_lcm = bpy . props . BoolProperty (
name = " use_lcm " ,
default = 0 ,
)
# SVD decode chunck
bpy . types . Scene . svd_decode_chunk_size = bpy . props . IntProperty (
name = " svd_decode_chunk_size " ,
default = 2 ,
min = 1 ,
max = 100 ,
)
# SVD motion_bucket_id
bpy . types . Scene . svd_motion_bucket_id = bpy . props . IntProperty (
name = " svd_motion_bucket_id " ,
default = 30 ,
min = 1 ,
max = 512 ,
)
for cls in classes :
bpy . utils . register_class ( cls )
# LoRA
bpy . types . Scene . lora_files = bpy . props . CollectionProperty ( type = LORABrowserFileItem )
bpy . types . Scene . lora_files_index = bpy . props . IntProperty ( name = " Index " , default = 0 )
bpy . types . Scene . lora_folder = bpy . props . StringProperty (
name = " Folder " ,
description = " Select a folder " ,
subtype = " DIR_PATH " ,
default = " " ,
update = update_folder_callback ,
)
def unregister ( ) :
for cls in classes :
bpy . utils . unregister_class ( cls )
del bpy . types . Scene . generate_movie_prompt
del bpy . types . Scene . generate_audio_prompt
del bpy . types . Scene . generate_movie_x
del bpy . types . Scene . generate_movie_y
del bpy . types . Scene . movie_num_inference_steps
del bpy . types . Scene . movie_num_batch
del bpy . types . Scene . movie_num_seed
del bpy . types . Scene . movie_use_random
del bpy . types . Scene . movie_num_guidance
del bpy . types . Scene . generatorai_typeselect
del bpy . types . Scene . movie_path
del bpy . types . Scene . image_path
del bpy . types . Scene . refine_sd
del bpy . types . Scene . generatorai_styles
del bpy . types . Scene . inpaint_selected_strip
del bpy . types . Scene . openpose_use_bones
del bpy . types . Scene . use_scribble_image
del bpy . types . Scene . blip_cond_subject
del bpy . types . Scene . blip_tgt_subject
del bpy . types . Scene . blip_subject_image
del bpy . types . Scene . lora_files
del bpy . types . Scene . lora_files_index
if __name__ == " __main__ " :
unregister ( )
register ( )
