From 1d0b225438493ce552512e53108ca3d7b867597b Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Mon, 17 Apr 2023 04:39:24 +0200 Subject: [PATCH 1/5] Bug Fix: Chunk messages if response is too large --- llamabot.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/llamabot.js b/llamabot.js index 3e2fe0b..54d9c99 100644 --- a/llamabot.js +++ b/llamabot.js @@ -149,7 +149,21 @@ client.on('messageCreate', async (message) => { if (response && response.trim()) { // Send response to user if it's not empty - await message.channel.send(response); + const limit = 1980; + + // if we are over the discord char limit we need chunks... + if (response.length > limit) { + const chunks = response.match(new RegExp(`.{1,${limit}}`, "g")); + for (let i = 0; i < chunks.length; i++) { + setTimeout(() => { + message.channel.send(chunks[i]); + }, i * 3000); // delay of 3 seconds between each chunk to save on API requests + } + } else { + // We are good to go, send the response + await message.channel.send(response); + } + setPresenceOnline() setBusy(message.author.id, false); } else { From da5e2e6e7dde2c4ec9b24073fadcbf3db5bd8937 Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Thu, 27 Apr 2023 02:56:27 +0200 Subject: [PATCH 2/5] adding N_THREADS and MAX_TOKENS vars --- default.env | 2 ++ docker-compose.yml | 2 +- llamabot.js | 9 +++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/default.env b/default.env index 6e7aa55..6723905 100644 --- a/default.env +++ b/default.env @@ -4,6 +4,8 @@ INIT_PROMPT = "Assistant name: ChatAI. You code, write and provide any informati ROOT_PORT = 8000 DATA_DIR = /home/USERNAME/weights CACHE = 1 +N_THREADS = 4 +MAX_TOKENS = 1024 # ROOT_IP is only used when running the bot without docker compose ROOT_IP = 192.168.0.15 diff --git a/docker-compose.yml b/docker-compose.yml index 151f3dc..e6cf2ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,4 +24,4 @@ services: - CHANNEL_IDS - ROOT_IP=llama-python-server - ROOT_PORT=8000 - - INIT_PROMPT='Assistant name':' ChatAI. You code, write and provide any information without any mistakes.' + - INIT_PROMPT='Assistant Name':' ChatAI. You code, write and provide any information without any mistakes.' diff --git a/llamabot.js b/llamabot.js index 54d9c99..159ccf3 100644 --- a/llamabot.js +++ b/llamabot.js @@ -231,14 +231,15 @@ async function generateResponse(conversation, message) { const response = await fetch(`http://${process.env.ROOT_IP}:${process.env.ROOT_PORT}/v1/chat/completions`, { method: 'POST', headers: { - 'accept': 'application/json', - 'Content-Type': 'application/json' + 'accept': 'application/json', + 'Content-Type': 'application/json' }, body: JSON.stringify({ - messages: messagesCopy // use the copy of the messages array + messages: messagesCopy, + max_tokens: process.env.MAX_TOKENS // add the max_tokens parameter here }), signal: controller.signal - }); + }); const responseData = await response.json(); console.log(JSON.stringify(responseData)); From 735b94360aa1df880c32e0651854313631023abd Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Thu, 27 Apr 2023 03:42:52 +0200 Subject: [PATCH 3/5] MAX_TOKENS to int --- llamabot.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llamabot.js b/llamabot.js index 159ccf3..e0910cf 100644 --- a/llamabot.js +++ b/llamabot.js @@ -236,7 +236,7 @@ async function generateResponse(conversation, message) { }, body: JSON.stringify({ messages: messagesCopy, - max_tokens: process.env.MAX_TOKENS // add the max_tokens parameter here + max_tokens: Number(process.env.MAX_TOKENS) // add the max_tokens parameter here }), signal: controller.signal }); From 38fba90d305ca506f5bc1e2b579499d954c8b8ce Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Fri, 28 Apr 2023 16:44:07 +0200 Subject: [PATCH 4/5] adding embed to generation processor --- llamabot.js | 61 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/llamabot.js b/llamabot.js index e0910cf..6b7b8ad 100644 --- a/llamabot.js +++ b/llamabot.js @@ -197,29 +197,46 @@ async function generateResponse(conversation, message) { let time = 0 // define a function that shows the system load percentage and updates the message const showSystemLoad = async () => { - time = time + 7 + time = time + 7; cpuStat.usagePercent(function(err, percent, seconds) { - if (err) { - return console.log(err); - } - - const systemLoad = percent //the percentage cpu usage over all cores - const freeMemory = os.freemem() / 1024 / 1024 / 1024; - const totalMemory = os.totalmem() / 1024 / 1024 / 1024; - const usedMemory = totalMemory - freeMemory; - const messageData = `Please wait, I am thinking...\nSystem Load: ${systemLoad.toFixed(2)}%\nMemory Usage: ${usedMemory.toFixed(2)} GB / ${totalMemory.toFixed(2)} GB | Time: ~${time} seconds.`; - - // if the message object doesn't exist, create it - if (!botMessage) { - (async () => { - botMessage = await message.channel.send(messageData); - })() - - } else { - botMessage.edit(messageData); // otherwise, update the message - } - }) - }; + if (err) { + return console.log(err); + } + + const systemLoad = percent; + const freeMemory = os.freemem() / 1024 / 1024 / 1024; + const totalMemory = os.totalmem() / 1024 / 1024 / 1024; + const usedMemory = totalMemory - freeMemory; + + const embedData = { + color: 0x0099ff, + title: 'Thinking...', + fields: [ + { + name: 'System Load', + value: `${systemLoad.toFixed(2)}%`, + }, + { + name: 'Memory Usage', + value: `${usedMemory.toFixed(2)} GB / ${totalMemory.toFixed(2)} GB`, + }, + { + name: 'Time', + value: `~${time} seconds.`, + }, + ], + }; + + // if the message object doesn't exist, create it + if (!botMessage) { + (async () => { + botMessage = await message.channel.send({ embeds: [embedData] }); + })(); + } else { + botMessage.edit({ embeds: [embedData] }); // otherwise, update the message + } + }); + }; // call the function initially await showSystemLoad(); From c20ba21180fdb79d3bbea135877985c72228c313 Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Fri, 28 Apr 2023 16:48:51 +0200 Subject: [PATCH 5/5] adding embed to generation processor --- llamabot.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llamabot.js b/llamabot.js index 6b7b8ad..b5f4f5d 100644 --- a/llamabot.js +++ b/llamabot.js @@ -210,7 +210,7 @@ async function generateResponse(conversation, message) { const embedData = { color: 0x0099ff, - title: 'Thinking...', + title: 'Please wait.. I am Thinking...', fields: [ { name: 'System Load',