snxraven/llama-cpp-python-djs-bot
1
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix token reducer

Browse Source
This commit is contained in:
Raven Scott 2023-08-21 21:15:10 -04:00
parent a980f3cd43
commit 1f3ff44317
1 changed files with 6 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
llamabot.js
View File

@ -454,7 +454,7 @@ async function generateResponse(conversation, message) {
// Remove older conversations if necessary // Remove older conversations if necessary
const maxLength = 1800; const maxLength = 1800;
const tolerance = 100; // allow for some flexibility in the token length const tolerance = 25; // allow for some flexibility in the token length
if (tokenLength > maxLength + tolerance) { if (tokenLength > maxLength + tolerance) {
const diff = tokenLength - (maxLength + tolerance); const diff = tokenLength - (maxLength + tolerance);
let removedTokens = 0; let removedTokens = 0;
@ -462,22 +462,19 @@ async function generateResponse(conversation, message) {
// Iterate over the messages in reverse order // Iterate over the messages in reverse order
for (let i = messagesCopy.length - 1; i >= 0; i--) { for (let i = messagesCopy.length - 1; i >= 0; i--) {
const message = messagesCopy[i]; const message = messagesCopy[i];
const messageTokens = countLlamaTokens(message); const messageTokens = countLlamaTokens([message]);
// Calculate the token length of the current message
const messageTokenLength = countLlamaTokens(messageTokens);
// Check if the current message plus the tokens in the message is less than or equal to the diff // Check if the current message plus the tokens in the message is less than or equal to the diff
if (removedTokens + messageTokenLength <= diff) { if (removedTokens + messageTokens <= diff) {
messagesCopy.splice(i, 1); messagesCopy.splice(i, 1);
removedTokens += messageTokenLength; removedTokens += messageTokens;
console.log(removedTokens + " removed \nAfter Resize: " + countLlamaTokens(messagesCopy)); console.log(removedTokens + " removed \nAfter Resize: " + countLlamaTokens(messagesCopy));
} else { } else {
// Remove more than one message if necessary to bring the total length below the maximum allowed length // Remove more than one message if necessary to bring the total length below the maximum allowed length
const messagesToRemove = Math.floor(diff / messageTokenLength); const messagesToRemove = Math.floor(diff / messageTokens);
for (let j = 0; j < messagesToRemove; j++) { for (let j = 0; j < messagesToRemove; j++) {
messagesCopy.splice(i, 1); messagesCopy.splice(i, 1);
removedTokens += messageTokenLength; removedTokens += messageTokens;
} }
break; break;
} }
@ -510,7 +507,6 @@ async function generateResponse(conversation, message) {
if (time > 2) { if (time > 2) {
await botMessage.delete() await botMessage.delete()
clearInterval(refreshInterval); clearInterval(refreshInterval);
clearTimeout(timeout);
botMessage = null; botMessage = null;
} }