Spaces:
Running
Running
Merge code
Browse files
app.py
CHANGED
@@ -304,7 +304,8 @@ def set_mp4_comments_imageio_ffmpeg(input_file, comments):
|
|
304 |
return False
|
305 |
|
306 |
@torch.no_grad()
|
307 |
-
def worker(input_image, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf):
|
|
|
308 |
def encode_prompt(prompt, n_prompt):
|
309 |
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
310 |
|
@@ -397,9 +398,10 @@ def worker(input_image, prompts, n_prompt, seed, resolution, total_second_length
|
|
397 |
rnd = torch.Generator("cpu").manual_seed(seed)
|
398 |
|
399 |
history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
|
|
|
400 |
history_pixels = None
|
401 |
|
402 |
-
history_latents = torch.cat([history_latents, start_latent
|
403 |
total_generated_latent_frames = 1
|
404 |
|
405 |
if enable_preview:
|
@@ -425,252 +427,35 @@ def worker(input_image, prompts, n_prompt, seed, resolution, total_second_length
|
|
425 |
return
|
426 |
|
427 |
indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
|
428 |
-
|
429 |
-
|
430 |
-
|
431 |
-
def post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream):
|
432 |
-
total_generated_latent_frames += int(generated_latents.shape[2])
|
433 |
-
history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
|
434 |
-
|
435 |
-
if not high_vram:
|
436 |
-
offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
|
437 |
-
load_model_as_complete(vae, target_device=gpu)
|
438 |
-
|
439 |
-
if history_pixels is None:
|
440 |
-
real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
|
441 |
-
history_pixels = vae_decode(real_history_latents, vae).cpu()
|
442 |
-
else:
|
443 |
-
section_latent_frames = latent_window_size * 2
|
444 |
-
overlapped_frames = latent_window_size * 4 - 3
|
445 |
-
|
446 |
-
real_history_latents = history_latents[:, :, -min(section_latent_frames, total_generated_latent_frames):, :, :]
|
447 |
-
history_pixels = soft_append_bcthw(history_pixels, vae_decode(real_history_latents, vae).cpu(), overlapped_frames)
|
448 |
-
|
449 |
-
if not high_vram:
|
450 |
-
unload_complete_models()
|
451 |
-
|
452 |
-
if enable_preview or section_index == total_latent_sections - 1:
|
453 |
-
output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
|
454 |
-
|
455 |
-
save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
|
456 |
-
|
457 |
-
print(f'Decoded. Current latent shape pixel shape {history_pixels.shape}')
|
458 |
-
|
459 |
-
stream.output_queue.push(('file', output_filename))
|
460 |
-
return [total_generated_latent_frames, history_latents, history_pixels]
|
461 |
-
|
462 |
-
for section_index in range(total_latent_sections):
|
463 |
-
if stream.input_queue.top() == 'end':
|
464 |
-
stream.output_queue.push(('end', None))
|
465 |
-
return
|
466 |
-
|
467 |
-
print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
|
468 |
-
|
469 |
-
if len(prompt_parameters) > 0:
|
470 |
-
[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0)
|
471 |
-
|
472 |
-
if not high_vram:
|
473 |
-
unload_complete_models()
|
474 |
-
move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
|
475 |
-
|
476 |
-
if use_teacache:
|
477 |
-
transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
|
478 |
-
else:
|
479 |
-
transformer.initialize_teacache(enable_teacache=False)
|
480 |
-
|
481 |
-
clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
|
482 |
-
clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
|
483 |
-
|
484 |
-
generated_latents = sample_hunyuan(
|
485 |
-
transformer=transformer,
|
486 |
-
sampler='unipc',
|
487 |
-
width=width,
|
488 |
-
height=height,
|
489 |
-
frames=latent_window_size * 4 - 3,
|
490 |
-
real_guidance_scale=cfg,
|
491 |
-
distilled_guidance_scale=gs,
|
492 |
-
guidance_rescale=rs,
|
493 |
-
# shift=3.0,
|
494 |
-
num_inference_steps=steps,
|
495 |
-
generator=rnd,
|
496 |
-
prompt_embeds=llama_vec,
|
497 |
-
prompt_embeds_mask=llama_attention_mask,
|
498 |
-
prompt_poolers=clip_l_pooler,
|
499 |
-
negative_prompt_embeds=llama_vec_n,
|
500 |
-
negative_prompt_embeds_mask=llama_attention_mask_n,
|
501 |
-
negative_prompt_poolers=clip_l_pooler_n,
|
502 |
-
device=gpu,
|
503 |
-
dtype=torch.bfloat16,
|
504 |
-
image_embeddings=image_encoder_last_hidden_state,
|
505 |
-
latent_indices=latent_indices,
|
506 |
-
clean_latents=clean_latents,
|
507 |
-
clean_latent_indices=clean_latent_indices,
|
508 |
-
clean_latents_2x=clean_latents_2x,
|
509 |
-
clean_latent_2x_indices=clean_latent_2x_indices,
|
510 |
-
clean_latents_4x=clean_latents_4x,
|
511 |
-
clean_latent_4x_indices=clean_latent_4x_indices,
|
512 |
-
callback=callback,
|
513 |
-
)
|
514 |
-
|
515 |
-
[total_generated_latent_frames, history_latents, history_pixels] = post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream)
|
516 |
-
except:
|
517 |
-
traceback.print_exc()
|
518 |
-
|
519 |
-
if not high_vram:
|
520 |
-
unload_complete_models(
|
521 |
-
text_encoder, text_encoder_2, image_encoder, vae, transformer
|
522 |
-
)
|
523 |
-
|
524 |
-
stream.output_queue.push(('end', None))
|
525 |
-
return
|
526 |
-
|
527 |
-
@torch.no_grad()
|
528 |
-
def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf):
|
529 |
-
def encode_prompt(prompt, n_prompt):
|
530 |
-
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
531 |
-
|
532 |
-
if cfg == 1:
|
533 |
-
llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
|
534 |
-
else:
|
535 |
-
llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
536 |
-
|
537 |
-
llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
|
538 |
-
llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
|
539 |
-
|
540 |
-
llama_vec = llama_vec.to(transformer.dtype)
|
541 |
-
llama_vec_n = llama_vec_n.to(transformer.dtype)
|
542 |
-
clip_l_pooler = clip_l_pooler.to(transformer.dtype)
|
543 |
-
clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
|
544 |
-
return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
|
545 |
-
|
546 |
-
total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
|
547 |
-
total_latent_sections = int(max(round(total_latent_sections), 1))
|
548 |
-
|
549 |
-
job_id = generate_timestamp()
|
550 |
-
|
551 |
-
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
|
552 |
-
|
553 |
-
try:
|
554 |
-
# Clean GPU
|
555 |
-
if not high_vram:
|
556 |
-
unload_complete_models(
|
557 |
-
text_encoder, text_encoder_2, image_encoder, vae, transformer
|
558 |
-
)
|
559 |
-
|
560 |
-
# Text encoding
|
561 |
-
|
562 |
-
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
|
563 |
-
|
564 |
-
if not high_vram:
|
565 |
-
fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
|
566 |
-
load_model_as_complete(text_encoder_2, target_device=gpu)
|
567 |
-
|
568 |
-
prompt_parameters = []
|
569 |
-
|
570 |
-
for prompt_part in prompts:
|
571 |
-
prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
|
572 |
-
|
573 |
-
# Processing input image
|
574 |
-
|
575 |
-
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
|
576 |
-
|
577 |
-
H, W, C = input_image.shape
|
578 |
-
height, width = find_nearest_bucket(H, W, resolution=resolution)
|
579 |
-
|
580 |
-
def get_start_latent(input_image, height, width, vae, gpu, image_encoder, high_vram):
|
581 |
-
input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
|
582 |
-
|
583 |
-
#Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
|
584 |
-
|
585 |
-
input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
|
586 |
-
input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
|
587 |
-
|
588 |
-
# VAE encoding
|
589 |
-
|
590 |
-
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
|
591 |
-
|
592 |
-
if not high_vram:
|
593 |
-
load_model_as_complete(vae, target_device=gpu)
|
594 |
-
|
595 |
-
start_latent = vae_encode(input_image_pt, vae)
|
596 |
-
|
597 |
-
# CLIP Vision
|
598 |
-
|
599 |
-
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
|
600 |
-
|
601 |
-
if not high_vram:
|
602 |
-
load_model_as_complete(image_encoder, target_device=gpu)
|
603 |
-
|
604 |
-
image_encoder_last_hidden_state = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder).last_hidden_state
|
605 |
-
|
606 |
-
return [start_latent, image_encoder_last_hidden_state]
|
607 |
-
|
608 |
-
[start_latent, image_encoder_last_hidden_state] = get_start_latent(input_image, height, width, vae, gpu, image_encoder, high_vram)
|
609 |
-
|
610 |
-
# Dtype
|
611 |
-
|
612 |
-
image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
|
613 |
-
|
614 |
-
# Sampling
|
615 |
-
|
616 |
-
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
|
617 |
-
|
618 |
-
rnd = torch.Generator("cpu").manual_seed(seed)
|
619 |
-
|
620 |
-
history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
|
621 |
-
history_pixels = None
|
622 |
-
|
623 |
-
history_latents = torch.cat([start_latent.to(history_latents), history_latents], dim=2)
|
624 |
-
total_generated_latent_frames = 1
|
625 |
-
|
626 |
-
if enable_preview:
|
627 |
-
def callback(d):
|
628 |
-
preview = d['denoised']
|
629 |
-
preview = vae_decode_fake(preview)
|
630 |
-
|
631 |
-
preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
|
632 |
-
preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
|
633 |
-
|
634 |
-
if stream.input_queue.top() == 'end':
|
635 |
-
stream.output_queue.push(('end', None))
|
636 |
-
raise KeyboardInterrupt('User ends the task.')
|
637 |
-
|
638 |
-
current_step = d['i'] + 1
|
639 |
-
percentage = int(100.0 * current_step / steps)
|
640 |
-
hint = f'Sampling {current_step}/{steps}'
|
641 |
-
desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30), Resolution: {height}px * {width}px. The video is being extended now ...'
|
642 |
-
stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
|
643 |
-
return
|
644 |
else:
|
645 |
-
|
646 |
-
|
647 |
-
|
648 |
-
indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
|
649 |
-
latent_indices, clean_latent_1x_indices, clean_latent_2x_indices, clean_latent_4x_indices, clean_latent_indices_start = indices.split([latent_window_size, 1, 2, 16, 1], dim=1)
|
650 |
-
clean_latent_indices = torch.cat([clean_latent_1x_indices, clean_latent_indices_start], dim=1)
|
651 |
|
652 |
def post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream):
|
653 |
total_generated_latent_frames += int(generated_latents.shape[2])
|
654 |
-
history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2)
|
655 |
|
656 |
if not high_vram:
|
657 |
offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
|
658 |
load_model_as_complete(vae, target_device=gpu)
|
659 |
|
660 |
if history_pixels is None:
|
661 |
-
real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :]
|
662 |
history_pixels = vae_decode(real_history_latents, vae).cpu()
|
663 |
else:
|
664 |
section_latent_frames = latent_window_size * 2
|
665 |
overlapped_frames = latent_window_size * 4 - 3
|
666 |
|
667 |
-
real_history_latents = history_latents[:, :, :min(section_latent_frames, total_generated_latent_frames), :, :]
|
668 |
-
history_pixels = soft_append_bcthw(vae_decode(real_history_latents, vae).cpu(), history_pixels, overlapped_frames)
|
669 |
|
670 |
if not high_vram:
|
671 |
unload_complete_models()
|
672 |
|
673 |
-
if enable_preview or section_index == 0:
|
674 |
output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
|
675 |
|
676 |
save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
|
@@ -680,7 +465,7 @@ def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_se
|
|
680 |
stream.output_queue.push(('file', output_filename))
|
681 |
return [total_generated_latent_frames, history_latents, history_pixels]
|
682 |
|
683 |
-
for section_index in range(total_latent_sections - 1, -1, -1):
|
684 |
if stream.input_queue.top() == 'end':
|
685 |
stream.output_queue.push(('end', None))
|
686 |
return
|
@@ -688,7 +473,7 @@ def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_se
|
|
688 |
print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
|
689 |
|
690 |
if len(prompt_parameters) > 0:
|
691 |
-
[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(len(prompt_parameters) - 1)
|
692 |
|
693 |
if not high_vram:
|
694 |
unload_complete_models()
|
@@ -699,8 +484,12 @@ def worker_last_frame(input_image, prompts, n_prompt, seed, resolution, total_se
|
|
699 |
else:
|
700 |
transformer.initialize_teacache(enable_teacache=False)
|
701 |
|
702 |
-
|
703 |
-
|
|
|
|
|
|
|
|
|
704 |
|
705 |
generated_latents = sample_hunyuan(
|
706 |
transformer=transformer,
|
@@ -791,7 +580,9 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
791 |
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Video processing ...'))))
|
792 |
|
793 |
# 20250506 pftq: Encode video
|
794 |
-
start_latent, input_image_np, video_latents, fps, height, width
|
|
|
|
|
795 |
|
796 |
# CLIP Vision
|
797 |
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
|
@@ -881,7 +672,7 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
881 |
if effective_clean_frames > 0 and split_idx < len(splits):
|
882 |
clean_latents_1x = splits[split_idx]
|
883 |
|
884 |
-
clean_latents = torch.cat([start_latent
|
885 |
|
886 |
# 20250507 pftq: Fix for <=1 sec videos.
|
887 |
max_frames = min(latent_window_size * 4 - 3, history_latents.shape[2] * 4)
|
@@ -900,7 +691,7 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
900 |
rnd = torch.Generator("cpu").manual_seed(seed)
|
901 |
|
902 |
# 20250506 pftq: Initialize history_latents with video latents
|
903 |
-
history_latents = video_latents
|
904 |
total_generated_latent_frames = history_latents.shape[2]
|
905 |
# 20250506 pftq: Initialize history_pixels to fix UnboundLocalError
|
906 |
history_pixels = None
|
@@ -1013,7 +804,7 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
|
|
1013 |
stream.output_queue.push(('end', None))
|
1014 |
return
|
1015 |
|
1016 |
-
def get_duration(input_image, image_position, prompt, generation_mode, n_prompt, randomize_seed, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf
|
1017 |
return total_second_length * 60 * (0.9 if use_teacache else 1.5) * (1 + ((steps - 25) / 100))
|
1018 |
|
1019 |
@spaces.GPU(duration=get_duration)
|
@@ -1034,8 +825,7 @@ def process(input_image,
|
|
1034 |
gpu_memory_preservation=6,
|
1035 |
enable_preview=True,
|
1036 |
use_teacache=False,
|
1037 |
-
mp4_crf=16
|
1038 |
-
progress = gr.Progress()
|
1039 |
):
|
1040 |
start = time.time()
|
1041 |
global stream
|
@@ -1060,7 +850,7 @@ def process(input_image,
|
|
1060 |
|
1061 |
stream = AsyncStream()
|
1062 |
|
1063 |
-
async_run(
|
1064 |
|
1065 |
output_filename = None
|
1066 |
|
@@ -1073,7 +863,6 @@ def process(input_image,
|
|
1073 |
|
1074 |
if flag == 'progress':
|
1075 |
preview, desc, html = data
|
1076 |
-
progress(None, desc = desc)
|
1077 |
yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
|
1078 |
|
1079 |
if flag == 'end':
|
@@ -1090,13 +879,12 @@ def process(input_image,
|
|
1090 |
"You can upscale the result with RIFE. To make all your generated scenes consistent, you can then apply a face swap on the main character.", gr.update(interactive=True), gr.update(interactive=False)
|
1091 |
break
|
1092 |
|
1093 |
-
def get_duration_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch
|
1094 |
return total_second_length * 60 * (0.9 if use_teacache else 2.3) * (1 + ((steps - 25) / 100))
|
1095 |
|
1096 |
# 20250506 pftq: Modified process to pass clean frame count, etc from video_encode
|
1097 |
@spaces.GPU(duration=get_duration_video)
|
1098 |
-
def process_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch
|
1099 |
-
progress = gr.Progress()):
|
1100 |
start = time.time()
|
1101 |
global stream, high_vram
|
1102 |
|
@@ -1144,7 +932,6 @@ def process_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, re
|
|
1144 |
|
1145 |
if flag == 'progress':
|
1146 |
preview, desc, html = data
|
1147 |
-
progress(None, desc = desc)
|
1148 |
#yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
|
1149 |
yield output_filename, gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True) # 20250506 pftq: Keep refreshing the video in case it got hidden when the tab was in the background
|
1150 |
|
@@ -1234,7 +1021,7 @@ with block:
|
|
1234 |
generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Video Extension", "video"]], elem_id="generation-mode", label="Generation mode", value = "image")
|
1235 |
text_to_video_hint = gr.HTML("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
|
1236 |
input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
|
1237 |
-
image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=100, info='0=Video start; 100=Video end')
|
1238 |
input_video = gr.Video(sources='upload', label="Input Video", height=320)
|
1239 |
timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed")
|
1240 |
prompt_number = gr.Slider(label="Timed prompt number", minimum=0, maximum=1000, value=0, step=1, info='Prompts will automatically appear')
|
@@ -1394,6 +1181,26 @@ with block:
|
|
1394 |
False, # enable_preview
|
1395 |
True, # use_teacache
|
1396 |
16 # mp4_crf
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1397 |
]
|
1398 |
],
|
1399 |
run_on_click = True,
|
|
|
304 |
return False
|
305 |
|
306 |
@torch.no_grad()
|
307 |
+
def worker(input_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf):
|
308 |
+
is_last_frame = (image_position == 100)
|
309 |
def encode_prompt(prompt, n_prompt):
|
310 |
llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
|
311 |
|
|
|
398 |
rnd = torch.Generator("cpu").manual_seed(seed)
|
399 |
|
400 |
history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
|
401 |
+
start_latent = start_latent.to(history_latents)
|
402 |
history_pixels = None
|
403 |
|
404 |
+
history_latents = torch.cat([start_latent, history_latents] if is_last_frame else [history_latents, start_latent], dim=2)
|
405 |
total_generated_latent_frames = 1
|
406 |
|
407 |
if enable_preview:
|
|
|
427 |
return
|
428 |
|
429 |
indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
|
430 |
+
if is_last_frame:
|
431 |
+
latent_indices, clean_latent_1x_indices, clean_latent_2x_indices, clean_latent_4x_indices, clean_latent_indices_start = indices.split([latent_window_size, 1, 2, 16, 1], dim=1)
|
432 |
+
clean_latent_indices = torch.cat([clean_latent_1x_indices, clean_latent_indices_start], dim=1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
433 |
else:
|
434 |
+
clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
|
435 |
+
clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
|
|
|
|
|
|
|
|
|
436 |
|
437 |
def post_process(generated_latents, total_generated_latent_frames, history_latents, high_vram, transformer, gpu, vae, history_pixels, latent_window_size, enable_preview, section_index, total_latent_sections, outputs_folder, mp4_crf, stream):
|
438 |
total_generated_latent_frames += int(generated_latents.shape[2])
|
439 |
+
history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2) if is_last_frame else torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
|
440 |
|
441 |
if not high_vram:
|
442 |
offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
|
443 |
load_model_as_complete(vae, target_device=gpu)
|
444 |
|
445 |
if history_pixels is None:
|
446 |
+
real_history_latents = history_latents[:, :, :total_generated_latent_frames, :, :] if is_last_frame else history_latents[:, :, -total_generated_latent_frames:, :, :]
|
447 |
history_pixels = vae_decode(real_history_latents, vae).cpu()
|
448 |
else:
|
449 |
section_latent_frames = latent_window_size * 2
|
450 |
overlapped_frames = latent_window_size * 4 - 3
|
451 |
|
452 |
+
real_history_latents = history_latents[:, :, :min(section_latent_frames, total_generated_latent_frames), :, :] if is_last_frame else history_latents[:, :, -min(section_latent_frames, total_generated_latent_frames):, :, :]
|
453 |
+
history_pixels = soft_append_bcthw(vae_decode(real_history_latents, vae).cpu(), history_pixels, overlapped_frames) if is_last_frame else soft_append_bcthw(history_pixels, vae_decode(real_history_latents, vae).cpu(), overlapped_frames)
|
454 |
|
455 |
if not high_vram:
|
456 |
unload_complete_models()
|
457 |
|
458 |
+
if enable_preview or section_index == (0 if is_last_frame else (total_latent_sections - 1)):
|
459 |
output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
|
460 |
|
461 |
save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
|
|
|
465 |
stream.output_queue.push(('file', output_filename))
|
466 |
return [total_generated_latent_frames, history_latents, history_pixels]
|
467 |
|
468 |
+
for section_index in range(total_latent_sections - 1, -1, -1) if is_last_frame else range(total_latent_sections):
|
469 |
if stream.input_queue.top() == 'end':
|
470 |
stream.output_queue.push(('end', None))
|
471 |
return
|
|
|
473 |
print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
|
474 |
|
475 |
if len(prompt_parameters) > 0:
|
476 |
+
[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop((len(prompt_parameters) - 1) if is_last_frame else 0)
|
477 |
|
478 |
if not high_vram:
|
479 |
unload_complete_models()
|
|
|
484 |
else:
|
485 |
transformer.initialize_teacache(enable_teacache=False)
|
486 |
|
487 |
+
if is_last_frame:
|
488 |
+
clean_latents_1x, clean_latents_2x, clean_latents_4x = history_latents[:, :, :sum([1, 2, 16]), :, :].split([1, 2, 16], dim=2)
|
489 |
+
clean_latents = torch.cat([clean_latents_1x, start_latent], dim=2)
|
490 |
+
else:
|
491 |
+
clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
|
492 |
+
clean_latents = torch.cat([start_latent, clean_latents_1x], dim=2)
|
493 |
|
494 |
generated_latents = sample_hunyuan(
|
495 |
transformer=transformer,
|
|
|
580 |
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Video processing ...'))))
|
581 |
|
582 |
# 20250506 pftq: Encode video
|
583 |
+
start_latent, input_image_np, video_latents, fps, height, width = video_encode(input_video, resolution, no_resize, vae, vae_batch_size=vae_batch, device=gpu)[:6]
|
584 |
+
start_latent = start_latent.to(dtype=torch.float32).cpu()
|
585 |
+
video_latents = video_latents.cpu()
|
586 |
|
587 |
# CLIP Vision
|
588 |
stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
|
|
|
672 |
if effective_clean_frames > 0 and split_idx < len(splits):
|
673 |
clean_latents_1x = splits[split_idx]
|
674 |
|
675 |
+
clean_latents = torch.cat([start_latent, clean_latents_1x], dim=2)
|
676 |
|
677 |
# 20250507 pftq: Fix for <=1 sec videos.
|
678 |
max_frames = min(latent_window_size * 4 - 3, history_latents.shape[2] * 4)
|
|
|
691 |
rnd = torch.Generator("cpu").manual_seed(seed)
|
692 |
|
693 |
# 20250506 pftq: Initialize history_latents with video latents
|
694 |
+
history_latents = video_latents
|
695 |
total_generated_latent_frames = history_latents.shape[2]
|
696 |
# 20250506 pftq: Initialize history_pixels to fix UnboundLocalError
|
697 |
history_pixels = None
|
|
|
804 |
stream.output_queue.push(('end', None))
|
805 |
return
|
806 |
|
807 |
+
def get_duration(input_image, image_position, prompt, generation_mode, n_prompt, randomize_seed, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf):
|
808 |
return total_second_length * 60 * (0.9 if use_teacache else 1.5) * (1 + ((steps - 25) / 100))
|
809 |
|
810 |
@spaces.GPU(duration=get_duration)
|
|
|
825 |
gpu_memory_preservation=6,
|
826 |
enable_preview=True,
|
827 |
use_teacache=False,
|
828 |
+
mp4_crf=16
|
|
|
829 |
):
|
830 |
start = time.time()
|
831 |
global stream
|
|
|
850 |
|
851 |
stream = AsyncStream()
|
852 |
|
853 |
+
async_run(worker, input_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf)
|
854 |
|
855 |
output_filename = None
|
856 |
|
|
|
863 |
|
864 |
if flag == 'progress':
|
865 |
preview, desc, html = data
|
|
|
866 |
yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
|
867 |
|
868 |
if flag == 'end':
|
|
|
879 |
"You can upscale the result with RIFE. To make all your generated scenes consistent, you can then apply a face swap on the main character.", gr.update(interactive=True), gr.update(interactive=False)
|
880 |
break
|
881 |
|
882 |
+
def get_duration_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
883 |
return total_second_length * 60 * (0.9 if use_teacache else 2.3) * (1 + ((steps - 25) / 100))
|
884 |
|
885 |
# 20250506 pftq: Modified process to pass clean frame count, etc from video_encode
|
886 |
@spaces.GPU(duration=get_duration_video)
|
887 |
+
def process_video(input_video, prompt, n_prompt, randomize_seed, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
|
|
|
888 |
start = time.time()
|
889 |
global stream, high_vram
|
890 |
|
|
|
932 |
|
933 |
if flag == 'progress':
|
934 |
preview, desc, html = data
|
|
|
935 |
#yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
|
936 |
yield output_filename, gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True) # 20250506 pftq: Keep refreshing the video in case it got hidden when the tab was in the background
|
937 |
|
|
|
1021 |
generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Video Extension", "video"]], elem_id="generation-mode", label="Generation mode", value = "image")
|
1022 |
text_to_video_hint = gr.HTML("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
|
1023 |
input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
|
1024 |
+
image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=100, info='0=Video start; 100=Video end (lower quality)')
|
1025 |
input_video = gr.Video(sources='upload', label="Input Video", height=320)
|
1026 |
timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed")
|
1027 |
prompt_number = gr.Slider(label="Timed prompt number", minimum=0, maximum=1000, value=0, step=1, info='Prompts will automatically appear')
|
|
|
1181 |
False, # enable_preview
|
1182 |
True, # use_teacache
|
1183 |
16 # mp4_crf
|
1184 |
+
],
|
1185 |
+
[
|
1186 |
+
"./img_examples/Example4.webp", # input_image
|
1187 |
+
100, # image_position
|
1188 |
+
"A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
|
1189 |
+
"image", # generation_mode
|
1190 |
+
"Missing arm, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry", # n_prompt
|
1191 |
+
True, # randomize_seed
|
1192 |
+
42, # seed
|
1193 |
+
672, # resolution
|
1194 |
+
1, # total_second_length
|
1195 |
+
9, # latent_window_size
|
1196 |
+
25, # steps
|
1197 |
+
1.0, # cfg
|
1198 |
+
10.0, # gs
|
1199 |
+
0.0, # rs
|
1200 |
+
6, # gpu_memory_preservation
|
1201 |
+
False, # enable_preview
|
1202 |
+
False, # use_teacache
|
1203 |
+
16 # mp4_crf
|
1204 |
]
|
1205 |
],
|
1206 |
run_on_click = True,
|