style

app.py

@@ -1,28 +1,28 @@
 import gradio as gr
 
-from defaults import defaults_vanilla_single, defaults_bt_spam, defaults_bt_single, defaults_vanilla_spam
-from defaults import ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER
-from backend import send_single, send_spam, get_message_single, get_message_spam
+from backend import (get_message_single, get_message_spam, send_single,
+                     send_spam)
+from defaults import (ADDRESS_BETTERTRANSFORMER, ADDRESS_VANILLA,
+                      defaults_bt_single, defaults_bt_spam,
+                      defaults_vanilla_single, defaults_vanilla_spam)
 
 with gr.Blocks() as demo:
-    gr.Markdown("""
+    gr.Markdown(
+        """
     Let's try out TorchServe + BetterTransformer! This is some longer description This is some longer description This is some longer description")
     
     ## Inference using...
     """
     )
-    
+
     with gr.Row():
         with gr.Column(scale=50):
             gr.Markdown("### Vanilla Transformers + TorchServe")
 
             address_input_vanilla = gr.Textbox(
-                max_lines=1,
-                label="ip vanilla",
-                value=ADDRESS_VANILLA,
-                visible=False
+                max_lines=1, label="ip vanilla", value=ADDRESS_VANILLA, visible=False
             )
-            
+
             input_model_vanilla = gr.Textbox(
                 max_lines=1,
                 label="Text",
@@ -30,11 +30,19 @@ with gr.Blocks() as demo:
             )
 
             btn_single_vanilla = gr.Button("Send single text request")
-            output_single_vanilla = gr.Markdown(label="Output single vanilla", value=get_message_single(**defaults_vanilla_single))
-            
-            btn_spam_vanilla = gr.Button("Spam text requests (from sst2 validation set)") 
-            output_spam_vanilla = gr.Markdown(label="Output spam vanilla", value=get_message_spam(**defaults_vanilla_spam))
-            
+            output_single_vanilla = gr.Markdown(
+                label="Output single vanilla",
+                value=get_message_single(**defaults_vanilla_single),
+            )
+
+            btn_spam_vanilla = gr.Button(
+                "Spam text requests (from sst2 validation set)"
+            )
+            output_spam_vanilla = gr.Markdown(
+                label="Output spam vanilla",
+                value=get_message_spam(**defaults_vanilla_spam),
+            )
+
             btn_single_vanilla.click(
                 fn=send_single,
                 inputs=[input_model_vanilla, address_input_vanilla],
@@ -53,9 +61,9 @@ with gr.Blocks() as demo:
                 max_lines=1,
                 label="ip bettertransformer",
                 value=ADDRESS_BETTERTRANSFORMER,
-                visible=False
+                visible=False,
             )
-            
+
             input_model_bettertransformer = gr.Textbox(
                 max_lines=1,
                 label="Text",
@@ -63,17 +71,21 @@ with gr.Blocks() as demo:
             )
 
             btn_single_bt = gr.Button("Send single text request")
-            output_single_bt = gr.Markdown(label="Output single bt", value=get_message_single(**defaults_bt_single))
-            
+            output_single_bt = gr.Markdown(
+                label="Output single bt", value=get_message_single(**defaults_bt_single)
+            )
+
             btn_spam_bt = gr.Button("Spam text requests (from sst2 validation set)")
-            output_spam_bt = gr.Markdown(label="Output spam bt", value=get_message_spam(**defaults_bt_spam))
-                        
+            output_spam_bt = gr.Markdown(
+                label="Output spam bt", value=get_message_spam(**defaults_bt_spam)
+            )
+
             btn_single_bt.click(
                 fn=send_single,
                 inputs=[input_model_bettertransformer, address_input_bettertransformer],
                 outputs=output_single_bt,
             )
-            
+
             btn_spam_bt.click(
                 fn=send_spam,
                 inputs=[address_input_bettertransformer],
@@ -81,4 +93,4 @@ with gr.Blocks() as demo:
             )
 
 demo.queue(concurrency_count=1)
-demo.launch()
+demo.launch()

backend.py

@@ -1,10 +1,11 @@
 import json
 
-from .defaults import SPAM_N_REQUESTS, ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER, HEADERS
-from .utils import ElapsedFuturesSession
-
 from datasets import load_dataset
 
+from .defaults import (ADDRESS_BETTERTRANSFORMER, ADDRESS_VANILLA, HEADERS,
+                       SPAM_N_REQUESTS)
+from .utils import ElapsedFuturesSession
+
 data = load_dataset("glue", "sst2", split="validation")
 
 RETURN_MESSAGE_SINGLE = """
@@ -18,8 +19,11 @@ Inference statistics:
 * Padding ratio: 0.0 %
 """
 
-RETURN_MESSAGE_SPAM = """
-Processing """ + f"{SPAM_N_REQUESTS}" + """ inputs sent asynchronously. Grab a coffee.
+RETURN_MESSAGE_SPAM = (
+    """
+Processing """
+    + f"{SPAM_N_REQUESTS}"
+    + """ inputs sent asynchronously. Grab a coffee.
 
 Inference statistics:
 
@@ -29,37 +33,64 @@ Inference statistics:
 * Mean padding ratio: {3} %
 * Mean sequence length: {4} tokens
 """
+)
+
+
+def get_message_single(
+    status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency, **kwargs
+):
+    return RETURN_MESSAGE_SINGLE.format(
+        status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency
+    )
+
+
+def get_message_spam(
+    resolution_time,
+    mean_inference_latency,
+    mean_peak_gpu_memory,
+    mean_padding_ratio,
+    mean_sequence_length,
+    **kwargs,
+):
+    return RETURN_MESSAGE_SPAM.format(
+        resolution_time,
+        mean_inference_latency,
+        mean_peak_gpu_memory,
+        mean_padding_ratio,
+        mean_sequence_length,
+    )
 
-def get_message_single(status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency, **kwargs):
-    return RETURN_MESSAGE_SINGLE.format(status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency)
-    
-def get_message_spam(resolution_time, mean_inference_latency, mean_peak_gpu_memory, mean_padding_ratio, mean_sequence_length, **kwargs):
-    return RETURN_MESSAGE_SPAM.format(resolution_time, mean_inference_latency, mean_peak_gpu_memory, mean_padding_ratio, mean_sequence_length)
 
 SESSION = ElapsedFuturesSession()
 
+
 def send_single(input_model_vanilla, address: str):
     assert address in [ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER]
-    
-    promise = SESSION.post(address, headers=HEADERS, data=input_model_vanilla.encode("utf-8"))
+
+    promise = SESSION.post(
+        address, headers=HEADERS, data=input_model_vanilla.encode("utf-8")
+    )
 
     response = promise.result()  # resolve immediately
 
     status = response.status_code
-    
+
     response_text = json.loads(response.text)
     prediction = response_text[0]
     inf_latency = response_text[1]
     peak_gpu_memory = response_text[2]
     end_to_end_latency = response.elapsed
-    
-    return get_message_single(status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency)
+
+    return get_message_single(
+        status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency
+    )
+
 
 def send_spam(address: str):
     assert address in [ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER]
-    
+
     # data = "this is positive lol"  #TODO: use dynamic data with padding
-    
+
     assert SPAM_N_REQUESTS <= len(data)
 
     inp = data.shuffle().select(range(SPAM_N_REQUESTS))
@@ -67,35 +98,41 @@ def send_spam(address: str):
     resolution_time = 0
     mean_inference_latency = 0
     mean_peak_gpu_memory = 0
-    
+
     n_pads = 0
     n_elems = 0
     sequence_length = 0
 
     promises = []
-    
+
     for i in range(SPAM_N_REQUESTS):
         input_data = inp[i]["sentence"].encode("utf-8")
         promises.append(SESSION.post(address, headers=HEADERS, data=input_data))
-    
+
     for promise in promises:
         response = promise.result()
-        
+
         response_text = json.loads(response.text)
-        
+
         resolution_time = max(resolution_time, response.elapsed)
-        
+
         mean_inference_latency += response_text[1]
         mean_peak_gpu_memory += response_text[2]
         n_pads += response_text[3]
         n_elems += response_text[4]
         sequence_length += response_text[5]
-    
+
     mean_padding_ratio = f"{n_pads / n_elems * 100:.2f}"
     mean_sequence_length = sequence_length / SPAM_N_REQUESTS
-    
+
     resolution_time = round(resolution_time, 2)
     mean_inference_latency = round(mean_inference_latency / SPAM_N_REQUESTS, 2)
     mean_peak_gpu_memory = round(mean_peak_gpu_memory / SPAM_N_REQUESTS, 2)
-    
-    return get_message_spam(resolution_time, mean_inference_latency, mean_peak_gpu_memory, mean_padding_ratio, mean_sequence_length)
+
+    return get_message_spam(
+        resolution_time,
+        mean_inference_latency,
+        mean_peak_gpu_memory,
+        mean_padding_ratio,
+        mean_sequence_length,
+    )

utils.py

@@ -1,9 +1,9 @@
+import time
+
 from requests_futures.sessions import FuturesSession
 
-import time
 
 class ElapsedFuturesSession(FuturesSession):
-
     def request(self, method, url, hooks=None, *args, **kwargs):
         start = time.time()
         if hooks is None:
@@ -13,13 +13,14 @@ class ElapsedFuturesSession(FuturesSession):
             r.elapsed = round((time.time() - start) * 1000, 2)
 
         try:
-            if isinstance(hooks['response'], (list, tuple)):
+            if isinstance(hooks["response"], (list, tuple)):
                 # needs to be first so we don't time other hooks execution
-                hooks['response'].insert(0, timing)
+                hooks["response"].insert(0, timing)
             else:
-                hooks['response'] = [timing, hooks['response']]
+                hooks["response"] = [timing, hooks["response"]]
         except KeyError:
-            hooks['response'] = timing
+            hooks["response"] = timing
 
-        return super(ElapsedFuturesSession, self) \
-            .request(method, url, hooks=hooks, *args, **kwargs) 
+        return super(ElapsedFuturesSession, self).request(
+            method, url, hooks=hooks, *args, **kwargs
+        )