fix

app/layout.tsx

@@ -5,7 +5,7 @@ import './globals.css'
 const inter = Inter({ subsets: ['latin'] })
 
 export const metadata: Metadata = {
-  title: '🔗 URL Surfer 🏄‍♂️',
+  title: 'Functions Playground',
   description: 'Navigate to URLs and perform realtime similarity search',
 }
 

app/tools/search.ts

@@ -6,7 +6,7 @@ function createSearchApi({ apiKey }: { apiKey: string }) {
   const paramsSchema = z.object({
     input: z.string(),
   });
-  const name = 'Search API';
+  const name = 'searchApi';
   const description = 'A custom search engine. Useful for when you need to answer questions about current events. Input should be a search query. Outputs a JSON array of results.';
 
   const execute = async ({ input }: z.infer<typeof paramsSchema>) => {

app/tools/surfer.ts

@@ -5,7 +5,7 @@ function createUrlSurfer() {
   const paramsSchema = z.object({
     input: z.string(),
   });
-  const name = 'URL Surfer';
+  const name = 'surfer';
   const description = 'A custom URL navigator. Useful when a URL is provided with a question. Input should be a prompt with a URL. Outputs a JSON array of relevant results.';
 
   return new Tool(paramsSchema, name, description, {} as any).tool;

pages/api/functions/index.ts

@@ -1,5 +1,5 @@
 import { NextApiRequest, NextApiResponse } from 'next';
-import fetch from 'node-fetch';
+import fetch, { RequestInfo } from 'node-fetch';
 import { JSDOM } from 'jsdom';
 // @ts-ignore
 import pdfParse from 'pdf-parse';
@@ -7,7 +7,7 @@ import puppeteer from 'puppeteer';
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
 import { MemoryVectorStore } from 'langchain/vectorstores/memory';
 import { HuggingFaceTransformersEmbeddings } from "langchain/embeddings/hf_transformers";
-import { createSerpApi } from '../../../app/tools/search'
+import { createSearchApi } from '../../../app/tools/search'
 
 export const config = {
     api: {
@@ -33,15 +33,25 @@ const model = new HuggingFaceTransformersEmbeddings({
 const urlRegex = /(https?:\/\/[^\s]+)/g;
 
 const [serpApi] =
-  createSerpApi({
+  createSearchApi({
     apiKey: process.env.SERP_API_KEY || "",
 });
 
 const handleContentText = async (targetUrl: string) => {
   const response = await fetch(targetUrl);
+  const status = response.status;
   const contentType = response.headers.get('content-type') || '';
   let content;
-  if (contentType.includes('application/pdf')) {
+
+  if (status >= 400) {
+    // If status is 400 or greater, try using puppeteer
+    const browser = await puppeteer.launch();
+    const page = await browser.newPage();
+    await page.goto(targetUrl, { waitUntil: 'networkidle0' }); // waits for the network to be idle before considering the navigation to be finished.
+    content = await page.evaluate(() => document.body.innerText);
+    await browser.close();
+    return content;
+  } else if (contentType.includes('application/pdf')) {
     const buffer = await response.arrayBuffer();
     content = await extractTextFromPDF(buffer as any);
   } else if (contentType.includes('text/html')) {
@@ -50,20 +60,13 @@ const handleContentText = async (targetUrl: string) => {
     const scripts = dom.window.document.querySelectorAll('script, style');
     scripts.forEach(element => element.remove());
     content = dom.window.document.body.textContent || '';
-
-    if (!content.trim()) {
-      const browser = await puppeteer.launch();
-      const page = await browser.newPage();
-      await page.goto(targetUrl);
-      content = await page.evaluate(() => document.body.innerText);
-      await browser.close();
-    }
   } else {
     content = await response.text();
   }
-  return content;
+  return content.trim();
 }
 
+
 const surferEmbedApi = async ({ input }: any) => {
   const urls = input.match(urlRegex);
   const targetUrl = urls ? urls[0] : null;