stream

app.py

@@ -20,11 +20,13 @@ def translate(inputs):
     ids = tokenizer(inputs, max_length=512)
     len_pad = 512 - len(ids)
     if len_pad > 0:
-        ids = ids + [0 for x in range(len_pad)]
-    print(ids)  
+        ids = ids + [0 for x in range(len_pad)]  
     inputs = torch.tensor([ids]).to(torch.device('cpu'))
-    outputs = model.generate(inputs, max_length=512)
-    text = tokenizer.ids2text(outputs.tolist()[0])
+    outputs = model.generate(inputs, max_length=512, stream=True)
+    text = ''
+    for value in outputs:
+        text = tokenizer.ids2text(value.tolist()[0])
+        yield text
     return text
 
 demo = gr.Interface(

requirements.txt

@@ -2,4 +2,4 @@ transformers==4.36.2
 sentencepiece==0.1.99
 numpy==1.24.2
 torch==2.0.1
--e git+https://github.com/t3ijk/tow.git@a21644091cefd744ac1c8a87be4bae1e6a8c1898#egg=tow
+-e git+https://github.com/t3ijk/tow.git@a797812b942e38d6c87dda6e5ff059b4b88e206f#egg=tow