update

.gitattributes

@@ -1,34 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

configs/v1-inference.yaml

@@ -1,70 +0,0 @@
-model:
-  base_learning_rate: 1.0e-04
-  target: ldm.models.diffusion.ddpm.LatentDiffusion
-  params:
-    linear_start: 0.00085
-    linear_end: 0.0120
-    num_timesteps_cond: 1
-    log_every_t: 200
-    timesteps: 1000
-    first_stage_key: "jpg"
-    cond_stage_key: "txt"
-    image_size: 64
-    channels: 4
-    cond_stage_trainable: false   # Note: different from the one we trained before
-    conditioning_key: crossattn
-    monitor: val/loss_simple_ema
-    scale_factor: 0.18215
-    use_ema: False
-
-    scheduler_config: # 10000 warmup steps
-      target: ldm.lr_scheduler.LambdaLinearScheduler
-      params:
-        warm_up_steps: [ 10000 ]
-        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
-        f_start: [ 1.e-6 ]
-        f_max: [ 1. ]
-        f_min: [ 1. ]
-
-    unet_config:
-      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
-      params:
-        image_size: 32 # unused
-        in_channels: 4
-        out_channels: 4
-        model_channels: 320
-        attention_resolutions: [ 4, 2, 1 ]
-        num_res_blocks: 2
-        channel_mult: [ 1, 2, 4, 4 ]
-        num_heads: 8
-        use_spatial_transformer: True
-        transformer_depth: 1
-        context_dim: 768
-        use_checkpoint: True
-        legacy: False
-
-    first_stage_config:
-      target: ldm.models.autoencoder.AutoencoderKL
-      params:
-        embed_dim: 4
-        monitor: val/rec_loss
-        ddconfig:
-          double_z: true
-          z_channels: 4
-          resolution: 256
-          in_channels: 3
-          out_ch: 3
-          ch: 128
-          ch_mult:
-          - 1
-          - 2
-          - 4
-          - 4
-          num_res_blocks: 2
-          attn_resolutions: []
-          dropout: 0.0
-        lossconfig:
-          target: torch.nn.Identity
-
-    cond_stage_config:
-      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder

text_to_animation/model.py

@@ -188,62 +188,3 @@ class ControlAnimationModel:
                         motion_field_strength_y=motion_field_strength_y,
                         ).images)[0]
         return utils.create_gif(np.array(vid), 4, path=None, watermark=None)
-
-
-    def generate_animation(
-        self,
-        prompt, #: str,
-        initial_frame_index, #: int,
-        input_video_path, #: str,
-        model_link = None,#: str = "dreamlike-art/dreamlike-photoreal-2.0",
-        motion_field_strength_x = 12,#: int = 12,
-        motion_field_strength_y= 12,#: int = 12,
-        t0= 44,#: int = 44,
-        t1= 47,#: int = 47,
-        n_prompt= "",#: str = "",
-        chunk_size= 8, #: int = 8,
-        video_length = 8, #: int = 8,
-        merging_ratio = 0., #: float = 0.0,
-        seed= 0,#: int = 0,
-        resolution=512,#: int = 512,
-        fps=2,#: int = 2,
-        use_cf_attn=True,#: bool = True,
-        use_motion_field=True,#: bool = True,
-        smooth_bg=False,#: bool = False,
-        smooth_bg_strength=0.4,#: float = 0.4,
-        path=None,#: str = None,
-    ):
-        video_path = gradio_utils.motion_to_video_path(input_video_path)
-
-        # added_prompt = 'best quality, HD, clay stop-motion, claymation, HQ, masterpiece, art, smooth'
-        # added_prompt = 'high quality, anatomically correct, clay stop-motion, aardman, claymation, smooth'
-        added_n_prompt = "longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer difits, cropped, worst quality, low quality, deformed body, bloated, ugly"
-        negative_prompts = added_n_prompt + ", " + n_prompt
-
-        video, fps = utils.prepare_video(
-            video_path, resolution, None, self.dtype, False, output_fps=4
-        )
-        control = utils.pre_process_pose(video, apply_pose_detect=False)
-        len_vid, _, h, w = video.shape
-        prng_seed = jax.random.PRNGKey(seed)
-        prompts = prompt
-        prompt_ids = self.pipe.prepare_text_inputs([prompts]*len_vid)
-        n_prompt_ids = self.pipe.prepare_text_inputs([negative_prompts]*len_vid)
-        prng = replicate_devices(prng_seed) #jax.random.split(prng, jax.device_count())
-        image = replicate_devices(control)
-        prompt_ids = replicate_devices(prompt_ids)
-        n_prompt_ids = replicate_devices(n_prompt_ids)
-        motion_field_strength_x = replicate_devices(jnp.array(motion_field_strength_x))
-        motion_field_strength_y = replicate_devices(jnp.array(motion_field_strength_y))
-        smooth_bg_strength = replicate_devices(jnp.array(smooth_bg_strength))
-        vid = (self.pipe(image=image,
-                        prompt_ids=prompt_ids,
-                        neg_prompt_ids=n_prompt_ids, 
-                        params=self.p_params,
-                        prng_seed=prng,
-                        jit = True,
-                        smooth_bg_strength=smooth_bg_strength,
-                        motion_field_strength_x=motion_field_strength_x,
-                        motion_field_strength_y=motion_field_strength_y,
-                        ).images)[0]
-        return utils.create_gif(np.array(vid), 4, path=None, watermark=None)

webui/app_control_animation.py

@@ -20,17 +20,6 @@ examples = [["A surfer in miami walking by the beach",
             None,
             0],
             ]
-# examples = [
-#     ["an astronaut waving the arm on the moon"],
-#     ["a sloth surfing on a wakeboard"],
-#     ["an astronaut walking on a street"],
-#     ["a cute cat walking on grass"],
-#     ["a horse is galloping on a street"],
-#     ["an astronaut is skiing down the hill"],
-#     ["a gorilla walking alone down the street"],
-#     ["a gorilla dancing on times square"],
-#     ["A panda dancing dancing like crazy on Times Square"],
-# ]
 
 def on_video_path_update(evt: gr.EventData):
     return f"Selection: **{evt._data}**"
@@ -209,17 +198,17 @@ def create_demo(model: ControlAnimationModel):
             seed,
         ]
 
-        # def submit_select(initial_frame_index: int):
-        #     if initial_frame_index != -1:  # More to next step
-        #         return {
-        #             frame_selection_view: gr.update(visible=False),
-        #             animation_view: gr.update(visible=True),
-        #         }
+        def submit_select(initial_frame_index: int):
+            if initial_frame_index != -1:  # More to next step
+                return {
+                    frame_selection_view: gr.update(visible=False),
+                    animation_view: gr.update(visible=True),
+                }
 
-        #     return {
-        #         frame_selection_view: gr.update(visible=True),
-        #         animation_view: gr.update(visible=False),
-        #     }
+            return {
+                frame_selection_view: gr.update(visible=True),
+                animation_view: gr.update(visible=False),
+            }
 
         gen_frames_button.click(
             fn=model.generate_initial_frames,
@@ -227,36 +216,22 @@ def create_demo(model: ControlAnimationModel):
             outputs=initial_frames,
         )
 
-        # gen_animation_button.click(
-        #     fn=submit_select,
-        #     inputs=initial_frame_index,
-        #     outputs=[frame_selection_view, animation_view],
-        # ).then(
-        #     fn=model.generate_animation,
-        #     inputs=animation_inputs,
-        #     outputs=result,
-        # )
-
         gen_animation_button.click(
+            fn=submit_select,
+            inputs=initial_frame_index,
+            outputs=[frame_selection_view, animation_view],
+        ).then(
             fn=model.generate_animation,
             inputs=animation_inputs,
             outputs=result,
         )
 
-        # gr.Examples(examples=examples,
-        #             inputs=inputs,
-        #             outputs=result,
-        #             fn=None,
-        #             run_on_click=False,
-        #             cache_examples=on_huggingspace,
-        # )
-
-        gr.Examples(examples=examples,
-                    inputs=animation_inputs,
-                    outputs=result,
-                    fn=model.generate_animation,
-                    cache_examples=on_huggingspace,
-                    run_on_click=True,
-                    )
+#         gr.Examples(examples=examples,
+#                     inputs=animation_inputs,
+#                     outputs=result,
+#                     fn=model.generate_animation,
+#                     cache_examples=on_huggingspace,
+#                     run_on_click=True,
+#                     )
 
     return demo