Update app.py

app.py

@@ -96,145 +96,11 @@ def run_gpt(
     return resp
 
 
-def compress_data(c,purpose, task, history):
-    seed=random.randint(1,1000000000)
-    print (c)
-    divr=int(c)/MAX_DATA
-    divi=int(divr)+1 if divr != int(divr) else int(divr)
-    chunk = int(int(c)/divr)
-    print(f'chunk:: {chunk}')
-    print(f'divr:: {divr}')
-    print (f'divi:: {divi}')
-    out = []
-    #out=""
-    s=0
-    e=chunk
-    print(f'e:: {e}')
-    new_history=""
-    task = f'Compile this data to fulfill the task: {task}, and complete the purpose: {purpose}\n'
-    for z in range(divi):
-        print(f's:e :: {s}:{e}')
-        
-        hist = history[s:e]
-        print(f'hist::\n{hist}')
-        resp = run_gpt(
-            COMPRESS_DATA_PROMPT,
-            stop_tokens=["observation:", "task:", "action:", "thought:"],
-            max_tokens=2048,
-            seed=seed,
-            purpose=purpose,
-            prefix_tog="normal",
-            task=task,
-            knowledge=new_history,
-            history=hist,
-        ).strip("\n")
-        new_history = resp
-        print (resp)
-        out+=resp
-        e=e+chunk
-        s=s+chunk
-    '''
-    resp = run_gpt(
-        COMPRESS_DATA_PROMPT,
-        stop_tokens=["observation:", "task:", "action:", "thought:"],
-        max_tokens=2048,
-        seed=seed,
-        purpose=purpose,
-        task=task,
-        knowledge=new_history,
-        history=result,
-    )
-    '''
-    print ("final" + resp)
-    history = "result: {}\n".format(resp)
-    return history
-
-
-
-def get_records(inp,data):
-    key_box=[]
-    seed=random.randint(1,1000000000)
-    print(inp)
-    out = str(data)
-    rl = len(out)
-    print(f'rl:: {rl}')
-    c=1
-    for i in str(out):
-        if i == " " or i=="," or i=="\n" or i=="/" or i=="." or i=="<":
-            c +=1
-    print (f'c:: {c}')
-    divr=int(c)/MAX_DATA
-    divi=int(divr)+1 if divr != int(divr) else int(divr)
-    chunk = int(int(c)/divr)
-    print(f'chunk:: {chunk}')
-    print(f'divr:: {divr}')
-    print (f'divi:: {divi}')
-    s=0
-    e=chunk
-    print(f'e:: {e}')
-    new_history=""
-    #task = f'Compile this data to fulfill the task: {task}, and complete the purpose: {purpose}\n'
-    for z in range(divi):
-        print(f's:e :: {s}:{e}')
-        
-        hist = out[s:e]
-        print(f'hist::\n{hist}')
-        resp = run_gpt(
-            GET_KEYWORD,
-            stop_tokens=[],
-            max_tokens=2048,
-            seed=seed,
-            purpose=inp,
-            prefix_tog="alternate",
-            task=inp,
-            knowledge=new_history,
-            history=hist,
-        ).strip("\n")
-        new_history = resp
-        print (f'resp {z}::\n {resp}')
-        #out+=resp
-        e=e+chunk
-        s=s+chunk
-        yield "", [(inp,new_history)]
-
-
-def get_key(inp,data):
-    key_box=[]
-    seed=random.randint(1,1000000000)
-    key_w = run_gpt(
-            GET_KEYWORD,
-            stop_tokens=[],
-            max_tokens=56,
-            seed=seed,
-            purpose=inp,
-            prefix_tog="normal",
-            task=inp,
-        ).split("<")[0]
-    print(f'key_w::{key_w}')
-    if " " in key_w:
-        key_w=key_w.split(" ")[-1]
-    for i,ba in enumerate(data):
-        each_key=data[i].keys()
-        print(each_key)
-        for z,zz in enumerate(list(each_key)[0]):
-            #for f,ff in enumerate(data[i][zz]):
-            ea = data[i][list(each_key)[0]][z]
-            try:
-                if ea['title'] and key_w in ea['title']:
-                    key_box.append(ea)
-                elif ea['description'] and key_w in ea['description']:
-                    key_box.append(ea)
-                elif ea['link'] and key_w in ea['link']:
-                    key_box.append(ea)
-            except Exception as e:
-                print(e)
-    print(key_box)
 
 NEWS_REPORTER="""You are an Expert News Aggregator.  Your duty is to read and compress all of the News Articles you are given into 10 or more individual articles that capture the full context of the current news.  Compile your articles into JSON format which the user will load into an RSS reader for other users to read.
-Output Format Example:
+*** Output Format Example:
 {output_format}
-
-News Articles:
+*** News Articles:
 {new_data}
 """
 output_format="""{"title": "title of the first article","description": "description of the article","article": "your custom written article","links": "all source links that have contributed to the article"},{"title": "title of the second article","description": "description of the article","article": "your custom written article","links": "all source links that have contributed to the article"}"""
@@ -285,17 +151,13 @@ def summarize(inp,history,data=None):
             new_data = out[s:e]
             #yield "", [(inp,f'{mes}\n{new_history}')]
             
-            content = NEWS_REPORTER.format(output_format=output_format,new_data=str(new_data.replace("{","").replace("}","")))
+            #content = NEWS_REPORTER.format(output_format=output_format,new_data=str(new_data.replace("{","").replace("}","")))
+            content = NEWS_REPORTER.format(output_format=output_format,new_data=str(new_data))
             stream = client.text_generation(content, **generate_kwargs, stream=True, details=True, return_full_text=False)
             for response in stream:
                 resp += response.token.text            
                 yield "", [(inp,resp)],None
             
-            #for line in resp.split("\n"):
-                #if 'title:' in line.lower():
-                    
-                
-                #out_json=""
             out_json=resp.replace("\n","").replace("```","")
             out_box.append(out_json.strip("</s>"))
             #out_box=eval(out_box)
@@ -304,16 +166,6 @@ def summarize(inp,history,data=None):
                            
             e=e+chunk
             s=s+chunk
-            
-            #history = "preliminary result: {}\n".format(resp)
-            #yield "", (inp,f'{mes}\n{history}')
-            #print ("final" + resp)
-            #out_hist = "result:\n{}".format(resp)
-            #return history
-            #yield "", [(inp,out_hist)]
-        
-            #out = str(out_hist)
-            #rawp = out
     else:
         rawp = "Provide a valid data source"
         history.append((inp,rawp))