Update app.py

app.py

@@ -14,6 +14,7 @@ import uuid
 #from query import tasks
 from agent import (
     PREFIX,
+    SAVE_MEMORY,
     COMPRESS_DATA_PROMPT,
     COMPRESS_DATA_PROMPT_SMALL,
     LOG_PROMPT,
@@ -24,6 +25,10 @@ api=HfApi()
 client = InferenceClient(
     "mistralai/Mixtral-8x7B-Instruct-v0.1"
 )
+reponame="Omnibus/tmp"
+save_data=f'https://huggingface.co/datasets/{reponame}/raw/main/'
+token_self = os.environ['HF_TOKEN']
+api=HfApi(token=token_self)
 
 def find_all(url):
     return_list=[]
@@ -289,7 +294,7 @@ def compress_data_og(c, instruct, history):
 
 
 
-def summarize(inp,history,report_check,data=None,files=None,url=None,pdf_url=None,pdf_batch=None):
+def summarize(inp,history,report_check,sum_mem_check,data=None,files=None,url=None,pdf_url=None,pdf_batch=None):
     json_box=[]
     if inp == "":
         inp = "Process this data"
@@ -313,9 +318,6 @@ def summarize(inp,history,report_check,data=None,files=None,url=None,pdf_url=Non
         except Exception as e:
             print(e)
             #data=f'{data}\nError reading URL ({batch_url})'
-            
-
-        
     if pdf_url.startswith("http"):
         print("PDF_URL")
         out = read_pdf_online(pdf_url)
@@ -352,41 +354,170 @@ def summarize(inp,history,report_check,data=None,files=None,url=None,pdf_url=Non
             if i == " " or i=="," or i=="\n":
                 c +=1
         print (f'c:: {c}')
-        
-        json_out = compress_data(c,inp,out)  
-        #json_box.append(json_out)
-
-        #json_object = json.dumps(eval(json_out), indent=4)
-        #json_box.append(json_out)
-        print(f'JSON_BOX:: {json_out}')
-        # Writing to sample.json
-        #with open("tmp.json", "w") as outfile:
-        #    outfile.write(json_object)
-        #outfile.close()
-        
-        #json_box.append(json_out)
-        out = str(json_out)
-        if report_check:
-            rl = len(out)
-            print(f'rl:: {rl}')
-            c=1
-            for i in str(out):
-                if i == " " or i=="," or i=="\n":
-                    c +=1
-            print (f'c2:: {c}')
-            rawp = compress_data_og(c,inp,out)
-        else:
-            rawp = out
+        if sum_mem_check=="Memory":
+            save_memory(inp,out)
+            rawp = "Complete"
+        if sum_mem_check=="Summarize":
+            json_out = compress_data(c,inp,out)  
+    
+            out = str(json_out)
+            if report_check:
+                rl = len(out)
+                print(f'rl:: {rl}')
+                c=1
+                for i in str(out):
+                    if i == " " or i=="," or i=="\n":
+                        c +=1
+                print (f'c2:: {c}')
+                rawp = compress_data_og(c,inp,out)
+            else:
+                rawp = out
     else:
         rawp = "Provide a valid data source"
-    #print (rawp)
-    #print (f'out:: {out}')
-    #history += "observation: the search results are:\n {}\n".format(out)
-    #task = "complete?"
     history.clear()
     history.append((inp,rawp))
     yield "", history,error_box,json_out
+SAVE_MEMORY = """
+You are attempting to complete the task
+task: {task}
+Data:
+{history}
+Instructions:
+Compile and categorize the data above into a JSON dictionary string
+Include ALL text, datapoints, titles, descriptions, and source urls indexed into an easy to search JSON format
+Your final response should be only the final formatted JSON string enclosed in brackets, and nothing else.
+Required keys:
+"keywords":["short", "list", "of", "keywords", "relevant", "to", "this", "entry"]
+"title":"title of entry"
+"description":"description of entry"
+"content":"full content of data about entry"
+"url":"https://url.source"
+"""
+
+def save_memory(purpose, history):
+    uid=uuid.uuid4()
+    history=str(history)
+    c=0
+    inp = str(history)
+    rl = len(inp)
+    print(f'rl:: {rl}')
+    for i in str(inp):
+        if i == " " or i=="," or i=="\n" or i=="/" or i=="." or i=="<":
+            c +=1
+    print (f'c:: {c}')
 
+    seed=random.randint(1,1000000000)
+    
+    print (c)
+    #tot=len(purpose)
+    #print(tot)
+    divr=int(c)/MAX_DATA
+    divi=int(divr)+1 if divr != int(divr) else int(divr)
+    chunk = int(int(c)/divr)
+    print(f'chunk:: {chunk}')
+    print(f'divr:: {divr}')
+    print (f'divi:: {divi}')
+    #out = []
+    #out=""
+    s=0
+    e=chunk
+    print(f'e:: {e}')
+    new_history=""
+    task = f'Index this Data\n'
+    for z in range(divi):
+        print(f's:e :: {s}:{e}')
+        
+        hist = inp[s:e]
+        
+        resp = run_gpt(
+            SAVE_MEMORY,
+            stop_tokens=["observation:", "task:", "action:", "thought:"],
+            max_tokens=4096,
+            seed=seed,
+            purpose=purpose,
+            task=task,
+            history=hist,
+        ).strip('\n')
+        #new_history = resp
+        print (resp)
+        #out+=resp
+        e=e+chunk
+        s=s+chunk
+        print ("final1" + resp)
+        try:
+            resp='[{'+resp.split('[{')[1].split('</s>')[0]
+            print ("final2\n" + resp)
+            print(f"keywords:: {resp['keywords']}")
+        except Exception as e:
+            resp = resp
+            print(e)
+        timestamp=str(datetime.datetime.now())
+        timename=timestamp.replace(" ","--").replace(":","-").replace(".","-")
+        json_object=resp
+        #json_object = json.dumps(out_box)
+        #json_object = json.dumps(out_box,indent=4)
+        with open(f"tmp-{uid}.json", "w") as outfile:
+            outfile.write(json_object)
+        api.upload_file(
+        path_or_fileobj=f"tmp-{uid}.json",
+        path_in_repo=f"/mem-test2/{timename}.json",
+        repo_id=reponame,
+        #repo_id=save_data.split('datasets/',1)[1].split('/raw',1)[0],
+        token=token_self,
+        repo_type="dataset",
+        )
+        lines = resp.strip().strip("\n").split("\n")
+        r = requests.get(f'{save_data}mem-test2/main.json') 
+        print(f'status code main:: {r.status_code}')
+        if r.status_code==200:
+            
+            lod = json.loads(r.text)
+            #lod = eval(lod)
+            print (f'lod:: {lod}')
+        else:
+            lod = []
+        for i,line in enumerate(lines):
+            key_box=[]
+            print(f'LINE:: {line}')
+            if ":" in line:
+                print(f'line:: {line}')
+            
+            if "keywords" in line[:16]:
+                print(f'trying:: {line}')
+                keyw=line.split(":")[1]
+                print (keyw)
+                print (keyw.split("[")[1].split("]")[0])
+                keyw=keyw.split("[")[1].split("]")[0]
+                for ea in keyw.split(","):
+                    s1=""
+                    ea=ea.strip().strip("\n")
+                    for ev in ea:
+                        if ev.isalnum():
+                            s1+=ev
+                        if ev == " ":
+                            s1+=ev
+                        #ea=s1
+                    print(s1)
+                    key_box.append(s1)
+                lod.append({"file_name":timename,"keywords":key_box})
+                json_object = json.dumps(lod, indent=4)
+                with open(f"tmp2-{uid}.json", "w") as outfile2:
+                    outfile2.write(json_object)
+                api.upload_file(
+                path_or_fileobj=f"tmp2-{uid}.json",
+                path_in_repo=f"/mem-test2/main.json",
+                repo_id=reponame,
+                #repo_id=save_data.split('datasets/',1)[1].split('/raw',1)[0],
+                token=token_self,
+                repo_type="dataset",
+                )
+
+    #return [resp]
+
+
+
+
+    
 #################################
 def clear_fn():
     return "",[(None,None)]
@@ -399,6 +530,7 @@ with gr.Blocks() as app:
             prompt=gr.Textbox(label = "Instructions (optional)")
         with gr.Column(scale=1):
             report_check=gr.Checkbox(label="Return Report", value=True)
+            sum_mem_check=gr.Radio(label="Output",choices=["Summary","Memory"])
             button=gr.Button()
         
         #models_dd=gr.Dropdown(choices=[m for m in return_list],interactive=True)
@@ -421,6 +553,6 @@ with gr.Blocks() as app:
     #text=gr.JSON()
     #inp_query.change(search_models,inp_query,models_dd)
     clear_btn.click(clear_fn,None,[prompt,chatbot])
-    go=button.click(summarize,[prompt,chatbot,report_check,data,file,url,pdf_url,pdf_batch],[prompt,chatbot,e_box,json_out])
+    go=button.click(summarize,[prompt,chatbot,report_check,sum_mem_check,data,file,url,pdf_url,pdf_batch],[prompt,chatbot,e_box,json_out])
     stop_button.click(None,None,None,cancels=[go])
 app.queue(default_concurrency_limit=20).launch(server_port=7860,show_api=False)