MAX_TOKENS to int

default.env

@@ -4,6 +4,8 @@ INIT_PROMPT = "Assistant name: ChatAI. You code, write and provide any informati
 ROOT_PORT = 8000
 DATA_DIR = /home/USERNAME/weights
 CACHE = 1
+N_THREADS = 4
+MAX_TOKENS = 1024
 
 # ROOT_IP is only used when running the bot without docker compose
 ROOT_IP = 192.168.0.15

docker-compose.yml

@@ -24,4 +24,4 @@ services:
       - CHANNEL_IDS
       - ROOT_IP=llama-python-server
       - ROOT_PORT=8000
-      - INIT_PROMPT='Assistant name':' ChatAI. You code, write and provide any information without any mistakes.'
+      - INIT_PROMPT='Assistant Name':' ChatAI. You code, write and provide any information without any mistakes.'

llamabot.js

@@ -149,7 +149,21 @@ client.on('messageCreate', async (message) => {
 
         if (response && response.trim()) {
             // Send response to user if it's not empty
-            await message.channel.send(response);
+            const limit = 1980;
+
+            // if we are over the discord char limit we need chunks...
+            if (response.length > limit) {
+              const chunks = response.match(new RegExp(`.{1,${limit}}`, "g"));
+              for (let i = 0; i < chunks.length; i++) {
+                setTimeout(() => {
+                  message.channel.send(chunks[i]);
+                }, i * 3000); // delay of 3 seconds between each chunk to save on API requests
+              }
+            } else {
+              // We are good to go, send the response
+              await message.channel.send(response);
+            }
+            
             setPresenceOnline()
             setBusy(message.author.id, false);
         } else {
@@ -217,14 +231,15 @@ async function generateResponse(conversation, message) {
         const response = await fetch(`http://${process.env.ROOT_IP}:${process.env.ROOT_PORT}/v1/chat/completions`, {
             method: 'POST',
             headers: {
-                'accept': 'application/json',
+              'accept': 'application/json',
-                'Content-Type': 'application/json'
+              'Content-Type': 'application/json'
             },
             body: JSON.stringify({
-                messages: messagesCopy // use the copy of the messages array
+              messages: messagesCopy,
+              max_tokens: Number(process.env.MAX_TOKENS) // add the max_tokens parameter here
             }),
             signal: controller.signal
-        });
+          });
 
         const responseData = await response.json();
         console.log(JSON.stringify(responseData));