ensure scraped results actually enter AIs memory

backend-server/backend-server.js

@@ -7,7 +7,7 @@ import cheerio from 'cheerio';
 import llamaTokenizer from 'llama-tokenizer-js';
 import googleIt from 'google-it';
 
-const prompt = process.env.PROMPT
+const prompt = process.env.PROMPT;
 const app = express();
 const port = 3000;
 
@@ -31,14 +31,12 @@ const getTimestamp = () => {
 // Middleware to track conversation history by CF-Connecting-IP
 app.use((req, res, next) => {
    const ip = req.headers['x-forwarded-for-id'] || req.headers['cf-connecting-ip'] || req.headers['x-forwarded-for'] || req.headers['x-real-ip'] || req.ip;
-   const name = req.headers['x-forwarded-for-name']
+   const name = req.headers['x-forwarded-for-name'];
-   const guild = req.headers['x-forwarded-for-guild']
+   const guild = req.headers['x-forwarded-for-guild'];
-
 
    req.clientIp = ip; // Store the IP in a request property
 
    if (!conversationHistory[req.clientIp]) {
-
       if (name) {
          if (guild) {
             console.log(`${getTimestamp()} [INFO] Incoming request from: ${req.clientIp} | ${name} within ${guild}`); // Log the IP address
@@ -59,13 +57,11 @@ app.use((req, res, next) => {
          console.log(`${getTimestamp()} [INFO] Incoming request from: ${req.clientIp}`); // Log the IP address
          console.log(`${getTimestamp()} [INFO] Initializing conversation history for new IP: ${req.clientIp}`);
 
-         conversationHistory[req.clientIp] = {
+         conversationHistory[req.clientIp] = [{
             role: 'system',
             content: prompt
-         }
+         }];
       }
-
-
    }
    next();
 });
@@ -282,7 +278,6 @@ app.post('/api/v1/chat', async (req, res) => {
                      content: "Fetched Info: " + response
                   });
 
-
                   console.log("A request for a new person was made. Response: " + response);
                } else {
                   console.log('Failed to fetch random user.');
@@ -305,10 +300,8 @@ app.post('/api/v1/chat', async (req, res) => {
          }
       }
 
-
       const whatServersRegex = /\bwhat\s+servers\b/gi;
 
-
       if (whatServersRegex.test(userMessage)) {
          try {
             const response = await fetch(`https://api.my-mc.link/list_all_servers/${process.env.PATH_KEY}/`, {
@@ -332,7 +325,7 @@ app.post('/api/v1/chat', async (req, res) => {
 
                conversationHistory[ip].push({
                   role: 'user',
-                  content: "Fetched Info: " + response
+                  content: "Fetched Info: " + responseMessage
                });
 
                console.log("A request for server information was made. Response: " + responseMessage);
@@ -345,10 +338,8 @@ app.post('/api/v1/chat', async (req, res) => {
          }
       }
 
-
       const myMcRegex = /\bmy-mc\b/gi;
 
-
       if (myMcRegex.test(userMessage)) {
          try {
             const response = await fetch('https://my-mc.link/wiki.json');
@@ -384,10 +375,8 @@ app.post('/api/v1/chat', async (req, res) => {
          }
       }
 
-
       const dlinuxRegex = /\bdlinux\b/gi;
 
-
       if (dlinuxRegex.test(userMessage)) {
          try {
             const response = await fetch('https://my-mc.link/dwiki.json');
@@ -426,86 +415,116 @@ app.post('/api/v1/chat', async (req, res) => {
 
       const searchRegex = /\b[Ss]earch\s+(.+)\b/;
       const searchMatch = userMessage.match(searchRegex);
-      
+
       if (searchMatch) {
-          const searchQuery = searchMatch[1];
+         const searchQuery = searchMatch[1];
-          console.log(`${getTimestamp()} [INFO] Detected search query in user message: ${searchQuery}`);
+         console.log(`${getTimestamp()} [INFO] Detected search query in user message: ${searchQuery}`);
-      
-          const options = {
-              query: searchQuery,
-              limit: 5,
-              disableConsole: true
-          };
-      
-          try {
-              googleIt(options).then(async results => {
-                  let searchResponse = `Search Query: ${searchQuery}\n`;
-                  searchResponse += `Top Google search results:\n`;
-      
-                  for (let i = 0; i < results.length; i++) {
-                      const result = results[i];
-                      searchResponse += `${i + 1}. ${result.title} - ${result.link}\n`;
-      
-                      try {
-                          const scrapeResult = await scrapeWebPage(result.link, 200);
-                          searchResponse += `Scraped Data: ${scrapeResult}\n`;
-                      } catch (scrapeErr) {
-                          console.error(`${getTimestamp()} [ERROR] Failed to scrape URL: ${result.link}`, scrapeErr);
-                          searchResponse += `Failed to scrape URL: ${result.link}\n`;
-                      }
-                  }
-      
-                  const lastMessageIndex = conversationHistory[ip].length - 1;
-                  if (lastMessageIndex >= 0) {
-                      conversationHistory[ip][lastMessageIndex].content += "\n" + searchResponse;
-                      console.log(`${getTimestamp()} [INFO] Processed search query: ${searchQuery}, response: ${searchResponse}`);
-                  } else {
-                      console.error(`${getTimestamp()} [ERROR] Conversation history is unexpectedly empty for: ${ip}`);
-                  }
-              }).catch(err => {
-                  console.error(`${getTimestamp()} [ERROR] Failed to perform Google search: ${searchQuery}`, err);
-                  return res.status(500).json({
-                      message: "An error occurred while performing Google search",
-                      error: err.message
-                  });
-              });
-          } catch (err) {
-              console.error(`${getTimestamp()} [ERROR] An unexpected error occurred:`, err);
-              return res.status(500).json({
-                  message: "An unexpected error occurred",
-                  error: err.message
-              });
-          }
-      }
-      
 
-      // End Plugins --- 
+         const options = {
+            query: searchQuery,
+            limit: 5,
+            disableConsole: true
+         };
 
-      console.log(`${getTimestamp()} [INFO] Sending request to llama API for response`);
+         try {
+            const results = await googleIt(options);
+            let searchResponse = `Search Query: ${searchQuery}\n`;
+            searchResponse += `Top Google search results:\n`;
+            let scrapedContent = '';
 
-      const sent = {
+            for (let i = 0; i < results.length; i++) {
-         model: 'model',
+               const result = results[i];
-         messages: conversationHistory[ip]
+               searchResponse += `${i + 1}. ${result.title} - ${result.link}\n`;
+
+               try {
+                  const scrapeResult = await scrapeWebPage(result.link, 400);
+                  searchResponse += `Scraped Data: ${scrapeResult}\n`;
+                  scrapedContent += `Scraped Data from ${result.link}:\n${scrapeResult}\n`;
+               } catch (scrapeErr) {
+                  console.error(`${getTimestamp()} [ERROR] Failed to scrape URL: ${result.link}`, scrapeErr);
+                  searchResponse += `Failed to scrape URL: ${result.link}\n`;
+                  scrapedContent += `Failed to scrape URL: ${result.link}\n`;
+               }
+            }
+
+            const lastMessageIndex = conversationHistory[ip].length - 1;
+            if (lastMessageIndex >= 0) {
+               conversationHistory[ip][lastMessageIndex].content += "\n" + searchResponse;
+               console.log(`${getTimestamp()} [INFO] Processed search query: ${searchQuery}, response: ${searchResponse}`);
+            } else {
+               console.error(`${getTimestamp()} [ERROR] Conversation history is unexpectedly empty for: ${ip}`);
+            }
+
+            if (scrapedContent) {
+               conversationHistory[ip].push({
+                  role: 'assistant',
+                  content: scrapedContent
+               });
+               console.log(`${getTimestamp()} [INFO] Added scraped content to conversation history for: ${ip}`);
+            }
+
+            // Now send the request to llama API after scraping is done
+            console.log(`${getTimestamp()} [INFO] Sending request to llama API for response`);
+
+            const sent = {
+               model: 'model',
+               messages: conversationHistory[ip]
+            };
+
+            //console.log(sent);
+            const llamaResponse = await fetch(`http://127.0.0.1:8002/v1/chat/completions`, {
+               method: 'POST',
+               headers: {
+                  'Content-Type': 'application/json'
+               },
+               body: JSON.stringify(sent)
+            });
+
+            const response = await llamaResponse.json();
+            const assistantMessage = response.choices[0].message;
+            conversationHistory[ip].push(assistantMessage);
+
+            console.log(`${getTimestamp()} [INFO] Received response from llama API`);
+            console.log(`${getTimestamp()} [DEBUG] Finish Reason: ${response.choices[0].finish_reason}`);
+            console.log(`${getTimestamp()} [STATS] Usage: prompt_tokens=${response.usage.prompt_tokens}, completion_tokens=${response.usage.completion_tokens}, total_tokens=${response.usage.total_tokens}`);
+
+            res.json(assistantMessage);
+         } catch (err) {
+            console.error(`${getTimestamp()} [ERROR] Failed to perform Google search: ${searchQuery}`, err);
+            return res.status(500).json({
+               message: "An error occurred while performing Google search",
+               error: err.message
+            });
+         }
+      } else {
+         // If no search plugin is used, send the request to llama API directly
+         console.log(`${getTimestamp()} [INFO] Sending request to llama API for response`);
+
+         const sent = {
+            model: 'model',
+            messages: conversationHistory[ip]
+         };
+
+         console.log(sent);
+         const llamaResponse = await fetch(`http://127.0.0.1:8002/v1/chat/completions`, {
+            method: 'POST',
+            headers: {
+               'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(sent)
+         });
+
+         const response = await llamaResponse.json();
+         const assistantMessage = response.choices[0].message;
+         conversationHistory[ip].push(assistantMessage);
+
+         console.log(`${getTimestamp()} [INFO] Received response from llama API`);
+         console.log(`${getTimestamp()} [DEBUG] Finish Reason: ${response.choices[0].finish_reason}`);
+         console.log(`${getTimestamp()} [STATS] Usage: prompt_tokens=${response.usage.prompt_tokens}, completion_tokens=${response.usage.completion_tokens}, total_tokens=${response.usage.total_tokens}`);
+
+         res.json(assistantMessage);
       }
 
-      //console.log(sent);
-      const llamaResponse = await fetch(`http://127.0.0.1:8002/v1/chat/completions`, {
-         method: 'POST',
-         headers: {
-            'Content-Type': 'application/json'
-         },
-         body: JSON.stringify(sent)
-      });
-
-      const response = await llamaResponse.json();
-      const assistantMessage = response.choices[0].message;
-      conversationHistory[ip].push(assistantMessage);
-
-      console.log(`${getTimestamp()} [INFO] Received response from llama API`); // ${assistantMessage.content}
-      console.log(`${getTimestamp()} [DEBUG] Finish Reason: ${response.choices[0].finish_reason}`);
-      console.log(`${getTimestamp()} [STATS] Usage: prompt_tokens=${response.usage.prompt_tokens}, completion_tokens=${response.usage.completion_tokens}, total_tokens=${response.usage.total_tokens}`);
-
-      res.json(assistantMessage);
    } catch (error) {
       console.error(`${getTimestamp()} [ERROR] An error occurred while handling chat request`, error);
       res.status(500).json({
@@ -557,4 +576,4 @@ app.post('/api/v1/reset-conversation', (req, res) => {
 
 app.listen(port, () => {
    console.log(`${getTimestamp()} [INFO] Server running at http://localhost:${port}`);
-});
+});