Update app.py

app.py

@@ -19,13 +19,64 @@ from agent import (
 )
 api=HfApi()
 
-
-
 client = InferenceClient(
     "mistralai/Mixtral-8x7B-Instruct-v0.1"
 )
 
+def find_all(url):
+    return_list=[]
+    print (url)
+    #if action_input in query.tasks:
+    print (f"trying URL:: {url}")        
+    try:
+        if url != "" and url != None:    
+            out = []
+            source = requests.get(url)
+            #source = urllib.request.urlopen(url).read()
+            soup = bs4.BeautifulSoup(source.content,'lxml')
+            # title of the page
+            print(soup.title)
+            # get attributes:
+            print(soup.title.name)
+            # get values:
+            print(soup.title.string)
+            # beginning navigation:
+            print(soup.title.parent.name)
+            #rawp.append([tag.name for tag in soup.find_all()] )
+            print([tag.name for tag in soup.find_all()])
+            rawp=(f'RAW TEXT RETURNED: {soup.text}')
+            out.append(rawp)
+            q=("a","p","span","content","article")
+            for p in soup.find_all(q):
+                out.append([{q:p.string,"parent":p.parent.name,"previous":[b for b in p.previous],"first-child":[b.name for b in p.children],"content":p}])
+            print (f'OUT :: {out}')
+            '''
+            c=0
+            out = str(out)
+            rl = len(out)
+            print(f'rl:: {rl}')
+            #for ea in out:
+            for i in str(out):
+                if i == " " or i=="," or i=="\n":
+                    c +=1
+            print (f'c:: {c}')
+            if rl > MAX_DATA:
+                print("compressing...")
+                rawp = compress_data(c,purpose,task,out)    
+            print (rawp)
+            print (f'out:: {out}')
+            '''
+            return True, rawp
+        else: 
+            return False, "Enter Valid URL"
+    except Exception as e:
+        print (e)
+        return False, f'Error: {e}'
 
+        #else:
+    #    history = "observation: The search query I used did not return a valid response"
+        
+    return "MAIN", None, history, task
 
 def read_txt(txt_path):
     text=""
@@ -154,13 +205,18 @@ def compress_data(c, instruct, history):
 
 
 
-def summarize(inp,history,data=None,file=None):
+def summarize(inp,history,data=None,file=None,url=None):
     if inp == "":
         inp = "Process this data"
-        
-    
     history = [(inp,"Working on it...")] if not history else history
     yield "",history
+    if url != "":
+        val, out = find_all(url)
+        if not val:
+            data="Error"
+            rawp = out
+        else:
+            rawp=out
     if file:
         try: 
             print (file)