# https://modelscope.cn/models/damo/text-to-video-synthesis/summary
bl_info = {
" name " : " Generative AI " ,
" author " : " tintwotin " ,
" version " : ( 1 , 2 ) ,
" blender " : ( 3 , 4 , 0 ) ,
" location " : " Video Sequence Editor > Sidebar > Generative AI " ,
" description " : " Generate media in the VSE " ,
" category " : " Sequencer " ,
}
import bpy , ctypes , random
from bpy . types import Operator , Panel , AddonPreferences
from bpy . props import StringProperty , BoolProperty , EnumProperty , IntProperty , FloatProperty
import site , platform
import subprocess
import sys , os , aud , re
import string
from os . path import dirname , realpath , isfile
import shutil
os_platform = platform . system ( ) # 'Linux', 'Darwin', 'Java', 'Windows'
def show_system_console ( show ) :
if os_platform == " Windows " :
# https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-showwindow
SW_HIDE = 0
SW_SHOW = 5
ctypes . windll . user32 . ShowWindow (
ctypes . windll . kernel32 . GetConsoleWindow ( ) , SW_SHOW #if show else SW_HIDE
)
def set_system_console_topmost ( top ) :
if os_platform == " Windows " :
# https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-setwindowpos
HWND_NOTOPMOST = - 2
HWND_TOPMOST = - 1
HWND_TOP = 0
SWP_NOMOVE = 0x0002
SWP_NOSIZE = 0x0001
SWP_NOZORDER = 0x0004
ctypes . windll . user32 . SetWindowPos (
ctypes . windll . kernel32 . GetConsoleWindow ( ) ,
HWND_TOP if top else HWND_NOTOPMOST ,
0 ,
0 ,
0 ,
0 ,
SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER ,
)
def split_and_recombine_text ( text , desired_length = 200 , max_length = 300 ) :
""" Split text it into chunks of a desired length trying to keep sentences intact. """
# normalize text, remove redundant whitespace and convert non-ascii quotes to ascii
text = re . sub ( r ' \ n \ n+ ' , ' \n ' , text )
text = re . sub ( r ' \ s+ ' , ' ' , text )
text = re . sub ( r ' [“”] ' , ' " ' , text )
rv = [ ]
in_quote = False
current = " "
split_pos = [ ]
pos = - 1
end_pos = len ( text ) - 1
def seek ( delta ) :
nonlocal pos , in_quote , current
is_neg = delta < 0
for _ in range ( abs ( delta ) ) :
if is_neg :
pos - = 1
current = current [ : - 1 ]
else :
pos + = 1
current + = text [ pos ]
if text [ pos ] == ' " ' :
in_quote = not in_quote
return text [ pos ]
def peek ( delta ) :
p = pos + delta
return text [ p ] if p < end_pos and p > = 0 else " "
def commit ( ) :
nonlocal rv , current , split_pos
rv . append ( current )
current = " "
split_pos = [ ]
while pos < end_pos :
c = seek ( 1 )
# do we need to force a split?
if len ( current ) > = max_length :
if len ( split_pos ) > 0 and len ( current ) > ( desired_length / 2 ) :
# we have at least one sentence and we are over half the desired length, seek back to the last split
d = pos - split_pos [ - 1 ]
seek ( - d )
else :
# no full sentences, seek back until we are not in the middle of a word and split there
while c not in ' !?., \n ' and pos > 0 and len ( current ) > desired_length :
c = seek ( - 1 )
commit ( )
# check for sentence boundaries
elif not in_quote and ( c in ' !? \n ' or ( c == ' . ' and peek ( 1 ) in ' \n ' ) ) :
# seek forward if we have consecutive boundary markers but still within the max length
while pos < len ( text ) - 1 and len ( current ) < max_length and peek ( 1 ) in ' !?., ' :
c = seek ( 1 )
split_pos . append ( pos )
if len ( current ) > = desired_length :
commit ( )
# treat end of quote as a boundary if its followed by a space or newline
elif in_quote and peek ( 1 ) == ' " ' and peek ( 2 ) in ' \n ' :
seek ( 2 )
split_pos . append ( pos )
rv . append ( current )
# clean up, remove lines with only whitespace or punctuation
rv = [ s . strip ( ) for s in rv ]
rv = [ s for s in rv if len ( s ) > 0 and not re . match ( r ' ^[ \ s \ .,;:!?]*$ ' , s ) ]
return rv
def closest_divisible_64 ( num ) :
# Determine the remainder when num is divided by 64
remainder = num % 64
# If the remainder is less than or equal to 32, return num - remainder,
# but ensure the result is not less than 64
if remainder < = 32 :
result = num - remainder
return max ( result , 192 )
# Otherwise, return num + (64 - remainder)
else :
return num + ( 64 - remainder )
def find_first_empty_channel ( start_frame , end_frame ) :
for ch in range ( 1 , len ( bpy . context . scene . sequence_editor . sequences_all ) + 1 ) :
for seq in bpy . context . scene . sequence_editor . sequences_all :
if (
seq . channel == ch
and seq . frame_final_start < end_frame
and ( seq . frame_final_start + seq . frame_final_duration ) > start_frame
) :
break
else :
return ch
return 1
def clean_filename ( filename ) :
filename = filename [ : 50 ]
valid_chars = " -_.() %s %s " % ( string . ascii_letters , string . digits )
clean_filename = " " . join ( c if c in valid_chars else " _ " for c in filename )
clean_filename = clean_filename . replace ( ' \n ' , ' ' )
clean_filename = clean_filename . replace ( ' \r ' , ' ' )
return clean_filename . strip ( )
def clean_path ( full_path ) :
name , ext = os . path . splitext ( full_path )
dir_path , filename = os . path . split ( name )
cleaned_filename = clean_filename ( filename )
new_filename = cleaned_filename + ext
i = 1
while os . path . exists ( os . path . join ( dir_path , new_filename ) ) :
name , ext = os . path . splitext ( new_filename )
new_filename = f " { name . rsplit ( ' ( ' , 1 ) [ 0 ] } ( { i } ) { ext } "
i + = 1
return os . path . join ( dir_path , new_filename )
def limit_string ( my_string ) :
if len ( my_string ) > 77 :
print ( " Warning: String is longer than 77 characters. Excessive string: " , my_string [ 77 : ] )
return my_string [ : 77 ]
else :
return my_string
def import_module ( self , module , install_module ) :
show_system_console ( True )
set_system_console_topmost ( True )
module = str ( module )
try :
exec ( " import " + module )
except ModuleNotFoundError :
app_path = site . USER_SITE
if app_path not in sys . path :
sys . path . append ( app_path )
pybin = sys . executable
self . report ( { " INFO " } , " Installing: " + module + " module. " )
print ( " Installing: " + module + " module " )
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
install_module ,
" --no-warn-script-location " ,
" --user " ,
]
)
try :
exec ( " import " + module )
except ModuleNotFoundError :
return False
return True
def install_modules ( self ) :
app_path = site . USER_SITE
if app_path not in sys . path :
sys . path . append ( app_path )
pybin = sys . executable
print ( " Ensuring: pip " )
try :
subprocess . call ( [ pybin , " -m " , " ensurepip " ] )
subprocess . call ( [ pybin , " -m " , " pip " , " install " , " --upgrade " , " pip " ] )
except ImportError :
pass
try :
exec ( " import torch " )
except ModuleNotFoundError :
app_path = site . USER_SITE
if app_path not in sys . path :
sys . path . append ( app_path )
pybin = sys . executable
self . report ( { " INFO " } , " Installing: torch module. " )
print ( " Installing: torch module " )
if os_platform == " Windows " :
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" torch " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu118 " ,
" --no-warn-script-location " ,
" --user " ,
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" torchvision " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu118 " ,
" --no-warn-script-location " ,
" --user " ,
]
)
subprocess . check_call (
[
pybin ,
" -m " ,
" pip " ,
" install " ,
" torchaudio " ,
" --index-url " ,
" https://download.pytorch.org/whl/cu118 " ,
" --no-warn-script-location " ,
" --user " ,
]
)
else :
import_module ( self , " torch " , " torch " )
import_module ( self , " torchvision " , " torchvision " )
import_module ( self , " torchaudio " , " torchaudio " )
if os_platform == ' Darwin ' :
import_module ( self , " sox " , " sox " )
else :
import_module ( self , " soundfile " , " PySoundFile " )
import_module ( self , " diffusers " , " diffusers " )
import_module ( self , " accelerate " , " accelerate " )
import_module ( self , " transformers " , " transformers " )
import_module ( self , " sentencepiece " , " sentencepiece " )
import_module ( self , " safetensors " , " safetensors " )
import_module ( self , " cv2 " , " opencv_python " )
import_module ( self , " scipy " , " scipy " )
import_module ( self , " xformers " , " xformers " )
import_module ( self , " bark " , " git+https://github.com/suno-ai/bark.git " )
import_module ( self , " IPython " , " IPython " )
subprocess . check_call ( [ pybin , " -m " , " pip " , " install " , " numpy " , " --upgrade " ] )
class GeneratorAddonPreferences ( AddonPreferences ) :
bl_idname = __name__
soundselect : EnumProperty (
name = " Sound " ,
items = {
( " ding " , " Ding " , " A simple bell sound " ) ,
( " coin " , " Coin " , " A Mario-like coin sound " ) ,
( " user " , " User " , " Load a custom sound file " ) ,
} ,
default = " ding " ,
)
default_folder = os . path . join (
os . path . dirname ( os . path . abspath ( __file__ ) ) , " sounds " , " *.wav "
)
if default_folder not in sys . path :
sys . path . append ( default_folder )
usersound : StringProperty (
name = " User " ,
description = " Load a custom sound from your computer " ,
subtype = " FILE_PATH " ,
default = default_folder ,
maxlen = 1024 ,
)
playsound : BoolProperty (
name = " Audio Notification " ,
default = True ,
)
movie_model_card : bpy . props . EnumProperty (
name = " Movie Model Card " ,
items = [
( " strangeman3107/animov-0.1.1 " , " Animov (448x384) " , " Animov " ) ,
( " strangeman3107/animov-512x " , " Animov (512x512) " , " Animov " ) ,
( " damo-vilab/text-to-video-ms-1.7b " , " Modelscope (256x256) " , " Modelscope " ) ,
] ,
default = " strangeman3107/animov-0.1.1 " ,
)
image_model_card : bpy . props . EnumProperty (
name = " Image Model Card " ,
items = [
( " runwayml/stable-diffusion-v1-5 " , " Stable Diffusion 1.5 (512x512) " , " Stable Diffusion 1.5 " ) ,
( " stabilityai/stable-diffusion-2 " , " Stable Diffusion 2 (768x768) " , " Stable Diffusion 2 " ) ,
( " DeepFloyd/IF-I-M-v1.0 " , " DeepFloyd/IF-I-M-v1.0 " , " DeepFloyd " ) ,
] ,
default = " stabilityai/stable-diffusion-2 " ,
)
audio_model_card : bpy . props . EnumProperty (
name = " Audio Model Card " ,
items = [
( " cvssp/audioldm-s-full-v2 " , " AudioLDM S Full v2 " , " AudioLDM Small Full v2 " ) ,
#("cvssp/audioldm", "AudioLDM", "AudioLDM"),
( " bark " , " Bark " , " Bark " ) ,
] ,
default = " bark " ,
)
hugginface_token : bpy . props . StringProperty (
name = " Hugginface Token " ,
default = " hugginface_token " ,
subtype = " PASSWORD " ,
)
def draw ( self , context ) :
layout = self . layout
box = layout . box ( )
box . operator ( " sequencer.install_generator " )
box . prop ( self , " movie_model_card " )
box . prop ( self , " image_model_card " )
if self . image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
row = box . row ( align = True )
row . prop ( self , " hugginface_token " )
row . operator ( " wm.url_open " , text = " " , icon = ' URL ' ) . url = " https://huggingface.co/settings/tokens "
box . prop ( self , " audio_model_card " )
row = box . row ( align = True )
row . label ( text = " Notification: " )
row . prop ( self , " playsound " , text = " " )
sub_row = row . row ( )
sub_row . prop ( self , " soundselect " , text = " " )
if self . soundselect == " user " :
sub_row . prop ( self , " usersound " , text = " " )
sub_row . operator ( " renderreminder.play_notification " , text = " " , icon = " PLAY " )
sub_row . active = self . playsound
class GENERATOR_OT_install ( Operator ) :
""" Install all dependencies """
bl_idname = " sequencer.install_generator "
bl_label = " Install Dependencies "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
install_modules ( self )
self . report (
{ " INFO " } ,
" Installation of dependencies is finished. " ,
)
return { " FINISHED " }
class GENERATOR_OT_sound_notification ( Operator ) :
""" Test your notification settings """
bl_idname = " renderreminder.play_notification "
bl_label = " Test Notification "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
if addon_prefs . playsound :
device = aud . Device ( )
def coinSound ( ) :
sound = aud . Sound ( " " )
handle = device . play (
sound . triangle ( 1000 )
. highpass ( 20 )
. lowpass ( 2000 )
. ADSR ( 0 , 0.5 , 1 , 0 )
. fadeout ( 0.1 , 0.1 )
. limit ( 0 , 1 )
)
handle = device . play (
sound . triangle ( 1500 )
. highpass ( 20 )
. lowpass ( 2000 )
. ADSR ( 0 , 0.5 , 1 , 0 )
. fadeout ( 0.2 , 0.2 )
. delay ( 0.1 )
. limit ( 0 , 1 )
)
def ding ( ) :
sound = aud . Sound ( " " )
handle = device . play (
sound . triangle ( 3000 )
. highpass ( 20 )
. lowpass ( 1000 )
. ADSR ( 0 , 0.5 , 1 , 0 )
. fadeout ( 0 , 1 )
. limit ( 0 , 1 )
)
if addon_prefs . soundselect == " ding " :
ding ( )
if addon_prefs . soundselect == " coin " :
coinSound ( )
if addon_prefs . soundselect == " user " :
file = str ( addon_prefs . usersound )
if os . path . isfile ( file ) :
sound = aud . Sound ( file )
handle = device . play ( sound )
return { " FINISHED " }
class SEQEUNCER_PT_generate_ai ( Panel ) :
""" Generate Media using AI """
bl_idname = " SEQUENCER_PT_sequencer_generate_movie_panel "
bl_label = " Generative AI "
bl_space_type = " SEQUENCE_EDITOR "
bl_region_type = " UI "
bl_category = " Generative AI "
def draw ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
audio_model_card = addon_prefs . audio_model_card
layout = self . layout
layout . use_property_split = False
layout . use_property_decorate = False
scene = context . scene
type = scene . generatorai_typeselect
col = layout . column ( )
col . prop ( context . scene , " generatorai_typeselect " , text = " " )
layout = self . layout
col = layout . column ( align = True )
col . use_property_split = True
col . use_property_decorate = False
col . scale_y = 1.2
col . prop ( context . scene , " generate_movie_prompt " , text = " " , icon = " ADD " )
if type == " audio " and audio_model_card == " bark " :
pass
else :
col . prop ( context . scene , " generate_movie_negative_prompt " , text = " " , icon = " REMOVE " )
layout = self . layout
layout . use_property_split = True
layout . use_property_decorate = False
if type == " movie " or type == " image " :
col = layout . column ( align = True )
col . prop ( context . scene , " generate_movie_x " , text = " X " )
col . prop ( context . scene , " generate_movie_y " , text = " Y " )
col = layout . column ( align = True )
if type == " movie " or type == " image " :
col . prop ( context . scene , " generate_movie_frames " , text = " Frames " )
if type == " audio " and audio_model_card != " bark " :
col . prop ( context . scene , " audio_length_in_f " , text = " Frames " )
if type == " audio " and audio_model_card == " bark " :
col = layout . column ( align = True )
col . prop ( context . scene , " speakers " , text = " Speaker " )
col . prop ( context . scene , " languages " , text = " Language " )
else :
col . prop ( context . scene , " movie_num_inference_steps " , text = " Quality Steps " )
col . prop ( context . scene , " movie_num_guidance " , text = " Word Power " )
col = layout . column ( align = True )
row = col . row ( align = True )
sub_row = row . row ( align = True )
sub_row . prop ( context . scene , " movie_num_seed " , text = " Seed " )
row . prop ( context . scene , " movie_use_random " , text = " " , icon = " QUESTION " )
sub_row . active = not context . scene . movie_use_random
col . prop ( context . scene , " movie_num_batch " , text = " Batch Count " )
row = layout . row ( align = True )
row . scale_y = 1.1
if type == " movie " :
row . operator ( " sequencer.generate_movie " , text = " Generate " )
if type == " image " :
row . operator ( " sequencer.generate_image " , text = " Generate " )
if type == " audio " :
row . operator ( " sequencer.generate_audio " , text = " Generate " )
class SEQUENCER_OT_generate_movie ( Operator ) :
""" Generate Video """
bl_idname = " sequencer.generate_movie "
bl_label = " Prompt "
bl_description = " Convert text to video "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
if not scene . generate_movie_prompt :
self . report ( { " INFO " } , " Text prompt in the Generative AI tab is empty! " )
return { " CANCELLED " }
show_system_console ( True )
set_system_console_topmost ( True )
seq_editor = scene . sequence_editor
if not seq_editor :
scene . sequence_editor_create ( )
try :
import torch
from diffusers import DiffusionPipeline , DPMSolverMultistepScheduler
from diffusers . utils import export_to_video
except ModuleNotFoundError :
print ( " Dependencies needs to be installed in the add-on preferences. " )
self . report (
{ " INFO " } ,
" Dependencies needs to be installed in the add-on preferences. " ,
)
return { " CANCELLED " }
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
current_frame = scene . frame_current
prompt = scene . generate_movie_prompt
negative_prompt = scene . generate_movie_negative_prompt + " nsfw nude nudity "
movie_x = scene . generate_movie_x
movie_y = scene . generate_movie_y
x = scene . generate_movie_x = closest_divisible_64 ( movie_x )
y = scene . generate_movie_y = closest_divisible_64 ( movie_y )
duration = scene . generate_movie_frames
movie_num_inference_steps = scene . movie_num_inference_steps
movie_num_guidance = scene . movie_num_guidance
#wm = bpy.context.window_manager
#tot = scene.movie_num_batch
#wm.progress_begin(0, tot)
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
movie_model_card = addon_prefs . movie_model_card
# Options: https://huggingface.co/docs/diffusers/api/pipelines/text_to_video
pipe = DiffusionPipeline . from_pretrained (
movie_model_card ,
#"strangeman3107/animov-0.1.1",
#"damo-vilab/text-to-video-ms-1.7b",
torch_dtype = torch . float16 ,
variant = " fp16 " ,
)
pipe . scheduler = DPMSolverMultistepScheduler . from_config (
pipe . scheduler . config
)
# memory optimization
pipe . enable_model_cpu_offload ( )
pipe . enable_vae_slicing ( )
for i in range ( scene . movie_num_batch ) :
#wm.progress_update(i)
if i > 0 :
empty_channel = scene . sequence_editor . active_strip . channel
start_frame = (
scene . sequence_editor . active_strip . frame_final_start
+ scene . sequence_editor . active_strip . frame_final_duration
)
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
else :
empty_channel = find_first_empty_channel (
scene . frame_current ,
( scene . movie_num_batch * duration ) + scene . frame_current ,
)
start_frame = scene . frame_current
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( 0 , 999999 )
)
context . scene . movie_num_seed = seed
# Use cuda if possible
if torch . cuda . is_available ( ) :
generator = (
torch . Generator ( " cuda " ) . manual_seed ( seed ) if seed != 0 else None
)
else :
if seed != 0 :
generator = torch . Generator ( )
generator . manual_seed ( seed )
else :
generator = None
video_frames = pipe (
prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = movie_num_inference_steps ,
guidance_scale = movie_num_guidance ,
height = y ,
width = x ,
num_frames = duration ,
generator = generator ,
) . frames
# Move to folder
src_path = export_to_video ( video_frames )
dst_path = clean_path ( dirname ( realpath ( __file__ ) ) + " / " + os . path . basename ( src_path ) )
shutil . move ( src_path , dst_path )
# Add strip
if not os . path . isfile ( dst_path ) :
print ( " No resulting file found. " )
return { " CANCELLED " }
# strip = scene.sequence_editor.sequences.new_movie(
# name=context.scene.generate_movie_prompt + " " + str(seed),
# frame_start=start_frame,
# filepath=dst_path,
# channel=empty_channel,
# fit_method="FILL",
# )
for window in bpy . context . window_manager . windows :
screen = window . screen
for area in screen . areas :
if area . type == " SEQUENCE_EDITOR " :
from bpy import context
with context . temp_override ( window = window , area = area ) :
bpy . ops . sequencer . movie_strip_add ( filepath = dst_path ,
frame_start = start_frame ,
channel = empty_channel ,
fit_method = " FIT " ,
adjust_playback_rate = True ,
sound = False ,
use_framerate = False ,
)
strip = scene . sequence_editor . active_strip
strip . transform . filter = ' NEAREST '
scene . sequence_editor . active_strip = strip
strip . use_proxy = True
bpy . ops . sequencer . rebuild_proxy ( )
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip. Remove this if Blender crashes: https://docs.blender.org/api/current/info_gotcha.html#can-i-redraw-during-script-execution
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
break
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
bpy . ops . renderreminder . play_notification ( )
#wm.progress_end()
scene . frame_current = current_frame
return { " FINISHED " }
class SEQUENCER_OT_generate_audio ( Operator ) :
""" Generate Audio """
bl_idname = " sequencer.generate_audio "
bl_label = " Prompt "
bl_description = " Convert text to audio "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
if not scene . generate_movie_prompt :
self . report ( { " INFO " } , " Text prompt in the Generative AI tab is empty! " )
return { " CANCELLED " }
if not scene . sequence_editor :
scene . sequence_editor_create ( )
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
current_frame = scene . frame_current
prompt = scene . generate_movie_prompt
negative_prompt = scene . generate_movie_negative_prompt
movie_num_inference_steps = scene . movie_num_inference_steps
movie_num_guidance = scene . movie_num_guidance
audio_length_in_s = scene . audio_length_in_f / ( scene . render . fps / scene . render . fps_base )
try :
from diffusers import AudioLDMPipeline
import torch
import scipy
#from bark import SAMPLE_RATE, generate_audio, preload_models
from IPython . display import Audio
from scipy . io . wavfile import write as write_wav
import xformers
if addon_prefs . audio_model_card == " bark " :
os . environ [ " CUDA_VISIBLE_DEVICES " ] = " 0 "
import numpy as np
from bark . generation import (
generate_text_semantic ,
preload_models ,
)
from bark . api import semantic_to_waveform
from bark import generate_audio , SAMPLE_RATE
except ModuleNotFoundError :
print ( " Dependencies needs to be installed in the add-on preferences. " )
self . report (
{ " INFO " } ,
" Dependencies needs to be installed in the add-on preferences. " ,
)
return { " CANCELLED " }
show_system_console ( True )
set_system_console_topmost ( True )
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
if addon_prefs . audio_model_card != " bark " :
repo_id = addon_prefs . audio_model_card
pipe = AudioLDMPipeline . from_pretrained ( repo_id ) # , torch_dtype=torch.float16z
# Use cuda if possible
if torch . cuda . is_available ( ) :
pipe = pipe . to ( " cuda " )
else : #bark
preload_models (
text_use_small = True ,
coarse_use_small = True ,
fine_use_gpu = True ,
fine_use_small = True ,
)
for i in range ( scene . movie_num_batch ) :
#wm.progress_update(i)
if i > 0 :
empty_channel = scene . sequence_editor . active_strip . channel
start_frame = (
scene . sequence_editor . active_strip . frame_final_start
+ scene . sequence_editor . active_strip . frame_final_duration
)
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
else :
empty_channel = find_first_empty_channel (
scene . frame_current ,
100000000000000000000 ,
)
start_frame = scene . frame_current
if addon_prefs . audio_model_card == " bark " :
rate = 24000
GEN_TEMP = 0.6
SPEAKER = " v2/ " + scene . languages + " _ " + scene . speakers #"v2/"+
silence = np . zeros ( int ( 0.25 * rate ) ) # quarter second of silence
prompt = context . scene . generate_movie_prompt
prompt = prompt . replace ( " \n " , " " ) . strip ( )
sentences = split_and_recombine_text ( prompt , desired_length = 90 , max_length = 150 )
pieces = [ ]
for sentence in sentences :
print ( sentence )
semantic_tokens = generate_text_semantic (
sentence ,
history_prompt = SPEAKER ,
temp = GEN_TEMP ,
#min_eos_p=0.1, # this controls how likely the generation is to end
)
audio_array = semantic_to_waveform ( semantic_tokens , history_prompt = SPEAKER )
pieces + = [ audio_array , silence . copy ( ) ]
audio = np . concatenate ( pieces ) #Audio(np.concatenate(pieces), rate=rate)
filename = clean_path ( dirname ( realpath ( __file__ ) ) + " / " + prompt + " .wav " )
# Write the combined audio to a file
write_wav ( filename , rate , audio . transpose ( ) )
else : # AudioLDM
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( 0 , 999999 )
)
context . scene . movie_num_seed = seed
# Use cuda if possible
if torch . cuda . is_available ( ) :
generator = (
torch . Generator ( " cuda " ) . manual_seed ( seed ) if seed != 0 else None
)
else :
if seed != 0 :
generator = torch . Generator ( )
generator . manual_seed ( seed )
else :
generator = None
prompt = context . scene . generate_movie_prompt
# Options: https://huggingface.co/docs/diffusers/main/en/api/pipelines/audioldm
audio = pipe (
prompt ,
num_inference_steps = movie_num_inference_steps ,
audio_length_in_s = audio_length_in_s ,
guidance_scale = movie_num_guidance ,
generator = generator ,
) . audios [ 0 ]
rate = 16000
filename = clean_path ( dirname ( realpath ( __file__ ) ) + " / " + prompt + " .wav " )
write_wav ( filename , rate , audio . transpose ( ) ) #.transpose()
filepath = filename
if os . path . isfile ( filepath ) :
empty_channel = empty_channel
strip = scene . sequence_editor . sequences . new_sound (
name = prompt ,
filepath = filepath ,
channel = empty_channel ,
frame_start = start_frame ,
)
scene . sequence_editor . active_strip = strip
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip. Remove this if Blender crashes: https://docs.blender.org/api/current/info_gotcha.html#can-i-redraw-during-script-execution
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
else :
print ( " No resulting file found! " )
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
bpy . ops . renderreminder . play_notification ( )
return { " FINISHED " }
class SEQUENCER_OT_generate_image ( Operator ) :
""" Generate Image """
bl_idname = " sequencer.generate_image "
bl_label = " Prompt "
bl_description = " Convert text to image "
bl_options = { " REGISTER " , " UNDO " }
def execute ( self , context ) :
scene = context . scene
if scene . generate_movie_prompt == " " :
self . report ( { " INFO " } , " Text prompt in the Generative AI tab is empty! " )
return { " CANCELLED " }
show_system_console ( True )
set_system_console_topmost ( True )
scene = context . scene
seq_editor = scene . sequence_editor
if not seq_editor :
scene . sequence_editor_create ( )
try :
from diffusers import DiffusionPipeline , DPMSolverMultistepScheduler
from diffusers . utils import pt_to_pil
import torch
except ModuleNotFoundError :
print ( " Dependencies needs to be installed in the add-on preferences. " )
self . report (
{ " INFO " } ,
" Dependencies needs to be installed in the add-on preferences. " ,
)
return { " CANCELLED " }
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
current_frame = scene . frame_current
prompt = scene . generate_movie_prompt
negative_prompt = scene . generate_movie_negative_prompt + " nsfw nude nudity "
image_x = scene . generate_movie_x
image_y = scene . generate_movie_y
x = scene . generate_movie_x = closest_divisible_64 ( image_x )
y = scene . generate_movie_y = closest_divisible_64 ( image_y )
duration = scene . generate_movie_frames
image_num_inference_steps = scene . movie_num_inference_steps
image_num_guidance = scene . movie_num_guidance
#wm = bpy.context.window_manager
#tot = scene.movie_num_batch
#wm.progress_begin(0, tot)
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
image_model_card = addon_prefs . image_model_card
if image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
from huggingface_hub . commands . user import login
result = login ( token = addon_prefs . hugginface_token )
print ( " Login: " + str ( result ) )
# stage 1
stage_1 = DiffusionPipeline . from_pretrained ( " DeepFloyd/IF-I-M-v1.0 " , variant = " fp16 " , torch_dtype = torch . float16 )
stage_1 . enable_model_cpu_offload ( )
# stage 2
stage_2 = DiffusionPipeline . from_pretrained (
" DeepFloyd/IF-II-M-v1.0 " , text_encoder = None , variant = " fp16 " , torch_dtype = torch . float16
)
stage_2 . enable_model_cpu_offload ( )
# stage 3
safety_modules = {
" feature_extractor " : stage_1 . feature_extractor ,
" safety_checker " : stage_1 . safety_checker ,
" watermarker " : stage_1 . watermarker ,
}
stage_3 = DiffusionPipeline . from_pretrained (
" stabilityai/stable-diffusion-x4-upscaler " , * * safety_modules , torch_dtype = torch . float16
)
stage_3 . enable_model_cpu_offload ( )
else : # stable Diffusion
pipe = DiffusionPipeline . from_pretrained (
image_model_card ,
torch_dtype = torch . float16 ,
variant = " fp16 " ,
)
pipe . scheduler = DPMSolverMultistepScheduler . from_config ( pipe . scheduler . config )
# memory optimization
pipe . enable_model_cpu_offload ( )
pipe . enable_vae_slicing ( )
for i in range ( scene . movie_num_batch ) :
#wm.progress_update(i)
if i > 0 :
empty_channel = scene . sequence_editor . active_strip . channel
start_frame = (
scene . sequence_editor . active_strip . frame_final_start
+ scene . sequence_editor . active_strip . frame_final_duration
)
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
else :
empty_channel = find_first_empty_channel (
scene . frame_current ,
( scene . movie_num_batch * duration ) + scene . frame_current ,
)
start_frame = scene . frame_current
seed = context . scene . movie_num_seed
seed = (
seed
if not context . scene . movie_use_random
else random . randint ( 0 , 999999 )
)
context . scene . movie_num_seed = seed
# Use cuda if possible
if torch . cuda . is_available ( ) :
generator = (
torch . Generator ( " cuda " ) . manual_seed ( seed ) if seed != 0 else None
)
else :
if seed != 0 :
generator = torch . Generator ( )
generator . manual_seed ( seed )
else :
generator = None
if image_model_card == " DeepFloyd/IF-I-M-v1.0 " :
# stage 1
image = stage_1 (
prompt_embeds = prompt_embeds , negative_prompt_embeds = negative_embeds , generator = generator , output_type = " pt "
) . images
pt_to_pil ( image ) [ 0 ] . save ( " ./if_stage_I.png " )
# stage 2
image = stage_2 (
image = image ,
prompt_embeds = prompt_embeds ,
negative_prompt_embeds = negative_embeds ,
generator = generator ,
output_type = " pt " ,
) . images
pt_to_pil ( image ) [ 0 ] . save ( " ./if_stage_II.png " )
# stage 3
image = stage_3 ( prompt = prompt , image = image , noise_level = 100 , generator = generator ) . images
image [ 0 ] . save ( " ./if_stage_III.png " )
else : # Stable Diffusion
image = pipe (
prompt ,
negative_prompt = negative_prompt ,
num_inference_steps = image_num_inference_steps ,
guidance_scale = image_num_guidance ,
height = y ,
width = x ,
generator = generator ,
) . images [ 0 ]
# Move to folder
filename = clean_filename ( context . scene . generate_movie_prompt )
out_path = clean_path ( dirname ( realpath ( __file__ ) ) + " / " + filename + " .png " )
image . save ( out_path )
# Add strip
if os . path . isfile ( out_path ) :
strip = scene . sequence_editor . sequences . new_image (
name = context . scene . generate_movie_prompt + " " + str ( seed ) ,
frame_start = start_frame ,
filepath = out_path ,
channel = empty_channel ,
fit_method = " FIT " ,
)
strip . frame_final_duration = scene . generate_movie_frames
strip . transform . filter = ' NEAREST '
scene . sequence_editor . active_strip = strip
if i > 0 :
scene . frame_current = (
scene . sequence_editor . active_strip . frame_final_start
)
# Redraw UI to display the new strip. Remove this if Blender crashes: https://docs.blender.org/api/current/info_gotcha.html#can-i-redraw-during-script-execution
bpy . ops . wm . redraw_timer ( type = " DRAW_WIN_SWAP " , iterations = 1 )
else :
print ( " No resulting file found. " )
bpy . ops . renderreminder . play_notification ( )
#wm.progress_end()
scene . frame_current = current_frame
# clear the VRAM
if torch . cuda . is_available ( ) :
torch . cuda . empty_cache ( )
return { " FINISHED " }
class SEQUENCER_OT_strip_to_generatorAI ( Operator ) :
""" Convert selected text strips to Generative AI """
bl_idname = " sequencer.text_to_generator "
bl_label = " Convert Text Strips to Generative AI "
bl_options = { " INTERNAL " }
bl_description = " Adds selected text strips as Generative AI strips "
@classmethod
def poll ( cls , context ) :
return context . scene and context . scene . sequence_editor
def execute ( self , context ) :
preferences = context . preferences
addon_prefs = preferences . addons [ __name__ ] . preferences
play_sound = addon_prefs . playsound
addon_prefs . playsound = False
scene = context . scene
sequencer = bpy . ops . sequencer
sequences = bpy . context . sequences
strips = context . selected_sequences
prompt = scene . generate_movie_prompt
current_frame = scene . frame_current
type = scene . generatorai_typeselect
for strip in strips :
if strip . type == " TEXT " :
if strip . text :
print ( " Processing: " + strip . text )
scene . generate_movie_prompt = strip . text
scene . frame_current = strip . frame_final_start
if type == " movie " :
sequencer . generate_movie ( )
if type == " audio " :
sequencer . generate_audio ( )
if type == " image " :
sequencer . generate_image ( )
scene . frame_current = current_frame
context . scene . generate_movie_prompt = prompt
addon_prefs . playsound = play_sound
bpy . ops . renderreminder . play_notification ( )
return { " FINISHED " }
def panel_text_to_generatorAI ( self , context ) :
layout = self . layout
layout . separator ( )
layout . operator (
" sequencer.text_to_generator " , text = " Text to Generative AI " , icon = " SHADERFX "
)
classes = (
GeneratorAddonPreferences ,
SEQUENCER_OT_generate_movie ,
SEQUENCER_OT_generate_audio ,
SEQUENCER_OT_generate_image ,
SEQEUNCER_PT_generate_ai ,
GENERATOR_OT_sound_notification ,
SEQUENCER_OT_strip_to_generatorAI ,
GENERATOR_OT_install ,
)
def register ( ) :
bpy . types . Scene . generate_movie_prompt = bpy . props . StringProperty (
name = " generate_movie_prompt " , default = " "
)
bpy . types . Scene . generate_movie_negative_prompt = bpy . props . StringProperty (
name = " generate_movie_negative_prompt " ,
default = " text, watermark, copyright, blurry, grainy, copyright " ,
)
bpy . types . Scene . generate_audio_prompt = bpy . props . StringProperty (
name = " generate_audio_prompt " , default = " "
)
bpy . types . Scene . generate_movie_x = bpy . props . IntProperty (
name = " generate_movie_x " ,
default = 448 ,
step = 64 ,
min = 192 ,
max = 1024 ,
)
bpy . types . Scene . generate_movie_y = bpy . props . IntProperty (
name = " generate_movie_y " ,
default = 384 ,
step = 64 ,
min = 192 ,
max = 1024 ,
)
# The number of frames to be generated.
bpy . types . Scene . generate_movie_frames = bpy . props . IntProperty (
name = " generate_movie_frames " ,
default = 18 ,
min = 1 ,
max = 125 ,
)
# The number of denoising steps. More denoising steps usually lead to a higher quality audio at the expense of slower inference.
bpy . types . Scene . movie_num_inference_steps = bpy . props . IntProperty (
name = " movie_num_inference_steps " ,
default = 25 ,
min = 1 ,
max = 100 ,
)
# The number of videos to generate.
bpy . types . Scene . movie_num_batch = bpy . props . IntProperty (
name = " movie_num_batch " ,
default = 1 ,
min = 1 ,
max = 100 ,
)
# The seed number.
bpy . types . Scene . movie_num_seed = bpy . props . IntProperty (
name = " movie_num_seed " ,
default = 1 ,
min = 1 ,
max = 2147483647 ,
)
# The seed number.
bpy . types . Scene . movie_use_random = bpy . props . BoolProperty (
name = " movie_use_random " ,
default = 1 ,
)
# The seed number.
bpy . types . Scene . movie_num_guidance = bpy . props . FloatProperty (
name = " movie_num_guidance " ,
default = 15.0 ,
min = 1 ,
max = 100 ,
)
# The frame audio duration.
bpy . types . Scene . audio_length_in_f = bpy . props . IntProperty (
name = " audio_length_in_f " ,
default = 80 ,
min = 1 ,
max = 10000 ,
)
bpy . types . Scene . generatorai_typeselect = bpy . props . EnumProperty (
name = " Sound " ,
items = [
( " movie " , " Video " , " Generate Video " ) ,
( " image " , " Image " , " Generate Image " ) ,
( " audio " , " Audio " , " Generate Audio " ) ,
] ,
default = " movie " ,
)
bpy . types . Scene . speakers = bpy . props . EnumProperty (
name = " Speakers " ,
items = [
( " speaker_0 " , " Speaker 0 " , " " ) ,
( " speaker_1 " , " Speaker 1 " , " " ) ,
( " speaker_2 " , " Speaker 2 " , " " ) ,
( " speaker_3 " , " Speaker 3 " , " " ) ,
( " speaker_4 " , " Speaker 4 " , " " ) ,
( " speaker_5 " , " Speaker 5 " , " " ) ,
( " speaker_6 " , " Speaker 6 " , " " ) ,
( " speaker_7 " , " Speaker 7 " , " " ) ,
( " speaker_8 " , " Speaker 8 " , " " ) ,
( " speaker_9 " , " Speaker 9 " , " " ) ,
] ,
default = " speaker_6 " ,
)
bpy . types . Scene . languages = bpy . props . EnumProperty (
name = " Languages " ,
items = [
( " en " , " English " , " " ) ,
( " de " , " German " , " " ) ,
( " es " , " Spanish " , " " ) ,
( " fr " , " French " , " " ) ,
( " hi " , " Hindi " , " " ) ,
( " it " , " Italian " , " " ) ,
( " ja " , " Japanese " , " " ) ,
( " ko " , " Korean " , " " ) ,
( " pl " , " Polish " , " " ) ,
( " pt " , " Portuguese " , " " ) ,
( " ru " , " Russian " , " " ) ,
( " tr " , " Turkish " , " " ) ,
( " zh " , " Chinese, simplified " , " " ) ,
] ,
default = " en "
)
for cls in classes :
bpy . utils . register_class ( cls )
bpy . types . SEQUENCER_MT_add . append ( panel_text_to_generatorAI )
def unregister ( ) :
for cls in classes :
bpy . utils . unregister_class ( cls )
del bpy . types . Scene . generate_movie_prompt
del bpy . types . Scene . generate_audio_prompt
del bpy . types . Scene . generate_movie_x
del bpy . types . Scene . generate_movie_y
del bpy . types . Scene . movie_num_inference_steps
del bpy . types . Scene . movie_num_batch
del bpy . types . Scene . movie_num_seed
del bpy . types . Scene . movie_use_random
del bpy . types . Scene . movie_num_guidance
del bpy . types . Scene . generatorai_typeselect
bpy . types . SEQUENCER_MT_add . remove ( panel_text_to_generatorAI )
if __name__ == " __main__ " :
register ( )
