Update optimization.py

optimization.py

@@ -1,29 +1,16 @@
-from typing import Any, Callable, ParamSpec
-import spaces
 import torch
-from torch.utils._pytree import tree_map_only
-from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
-from optimization_utils import capture_component_call, aoti_compile, ZeroGPUCompiledModel, drain_module_parameters
-
-P = ParamSpec('P')
-
-TRANSFORMER_NUM_FRAMES_DIM = torch.export.Dim('num_frames', min=3, max=21)
-TRANSFORMER_DYNAMIC_SHAPES = {'hidden_states': {2: TRANSFORMER_NUM_FRAMES_DIM}}
-
-INDUCTOR_CONFIGS = {
-    'conv_1x1_as_mm': True,
-    'epilogue_fusion': False,
-    'coordinate_descent_tuning': True,
-    'coordinate_descent_check_all_directions': True,
-    'max_autotune': True,
-    'triton.cudagraphs': True,
-}
+import torchao
+from torchao.quantization import DEFAULT_INT4_AUTOQUANT_CLASS_LIST
 
 def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kwargs):
     print("[optimize_pipeline_] Starting pipeline optimization")
 
-    quantize_(pipeline.text_encoder, Int8WeightOnlyConfig())
-    print("[optimize_pipeline_] Text encoder quantized")
+    # Quantize and compile text encoder first (weight-only int8 quantization can be replaced by autoquant if preferred)
+    pipeline.text_encoder = torchao.autoquant(
+        torch.compile(pipeline.text_encoder, mode='max-autotune'),
+        qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST  # or remove for default quant
+    )
+    print("[optimize_pipeline_] Text encoder autoquantized and compiled")
 
     @spaces.GPU(duration=1500)
     def compile_transformer():
@@ -54,9 +41,16 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
         dynamic_shapes = tree_map_only((torch.Tensor, bool), lambda t: None, call.kwargs)
         dynamic_shapes |= TRANSFORMER_DYNAMIC_SHAPES
 
-        print("[compile_transformer] Quantizing transformers with Float8DynamicActivationFloat8WeightConfig")
-        quantize_(pipeline.transformer, Float8DynamicActivationFloat8WeightConfig())
-        quantize_(pipeline.transformer_2, Float8DynamicActivationFloat8WeightConfig())
+        # Use autoquant + torch.compile on transformers
+        print("[compile_transformer] Autoquantizing and compiling transformer")
+        compiled_transformer = torchao.autoquant(
+            torch.compile(pipeline.transformer, mode='max-autotune'),
+            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST,
+        )
+        compiled_transformer_2 = torchao.autoquant(
+            torch.compile(pipeline.transformer_2, mode='max-autotune'),
+            qtensor_class_list=DEFAULT_INT4_AUTOQUANT_CLASS_LIST,
+        )
 
         hidden_states: torch.Tensor = call.kwargs['hidden_states']
         hidden_states_transposed = hidden_states.transpose(-1, -2).contiguous()
@@ -68,53 +62,34 @@ def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kw
             hidden_states_landscape = hidden_states_transposed
             hidden_states_portrait = hidden_states
 
-        print("[compile_transformer] Exporting transformer landscape model")
-        exported_landscape_1 = torch.export.export(
-            mod=pipeline.transformer,
-            args=call.args,
-            kwargs={**call.kwargs, 'hidden_states': hidden_states_landscape},
-            dynamic_shapes=dynamic_shapes,
-        )
-        torch.cuda.synchronize()
-
-        print("[compile_transformer] Exporting transformer portrait model")
-        exported_portrait_2 = torch.export.export(
-            mod=pipeline.transformer_2,
-            args=call.args,
-            kwargs={**call.kwargs, 'hidden_states': hidden_states_portrait},
-            dynamic_shapes=dynamic_shapes,
-        )
-        torch.cuda.synchronize()
+        # Replace forward with quantized & compiled versions, wrapped for shape dispatch
+        def combined_transformer_1(*a, **k):
+            if k['hidden_states'].shape[-1] > k['hidden_states'].shape[-2]:
+                return compiled_transformer(*a, **k)
+            else:
+                # Swap hidden states for portrait? Use transpose if needed.
+                k_mod = k.copy()
+                k_mod['hidden_states'] = hidden_states_portrait
+                return compiled_transformer(*a, **k_mod)
 
-        print("[compile_transformer] Compiling models with AoT compilation")
-        compiled_landscape_1 = aoti_compile(exported_landscape_1, INDUCTOR_CONFIGS)
-        compiled_portrait_2 = aoti_compile(exported_portrait_2, INDUCTOR_CONFIGS)
+        def combined_transformer_2(*a, **k):
+            if k['hidden_states'].shape[-1] > k['hidden_states'].shape[-2]:
+                return compiled_transformer_2(*a, **k)
+            else:
+                k_mod = k.copy()
+                k_mod['hidden_states'] = hidden_states_portrait
+                return compiled_transformer_2(*a, **k_mod)
 
-        compiled_landscape_2 = ZeroGPUCompiledModel(compiled_landscape_1.archive_file, compiled_portrait_2.weights)
-        compiled_portrait_1 = ZeroGPUCompiledModel(compiled_portrait_2.archive_file, compiled_landscape_1.weights)
+        pipeline.transformer.forward = combined_transformer_1
+        drain_module_parameters(pipeline.transformer)
 
-        print("[compile_transformer] Compilation done")
-        return compiled_landscape_1, compiled_landscape_2, compiled_portrait_1, compiled_portrait_2
+        pipeline.transformer_2.forward = combined_transformer_2
+        drain_module_parameters(pipeline.transformer_2)
 
-    cl1, cl2, cp1, cp2 = compile_transformer()
-
-    def combined_transformer_1(*args, **kwargs):
-        hidden_states: torch.Tensor = kwargs['hidden_states']
-        if hidden_states.shape[-1] > hidden_states.shape[-2]:
-            return cl1(*args, **kwargs)
-        else:
-            return cp1(*args, **kwargs)
-
-    def combined_transformer_2(*args, **kwargs):
-        hidden_states: torch.Tensor = kwargs['hidden_states']
-        if hidden_states.shape[-1] > hidden_states.shape[-2]:
-            return cl2(*args, **kwargs)
-        else:
-            return cp2(*args, **kwargs)
+        print("[compile_transformer] Transformers autoquantized, compiled, and patched")
 
-    pipeline.transformer.forward = combined_transformer_1
-    drain_module_parameters(pipeline.transformer)
-    pipeline.transformer_2.forward = combined_transformer_2
-    drain_module_parameters(pipeline.transformer_2)
+        # Return compiled models for reference if needed
+        return compiled_transformer, compiled_transformer_2
 
+    cl1, cl2 = compile_transformer()
     print("[optimize_pipeline_] Optimization complete")