snxraven/llama-cpp-python-djs-bot
1
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix token reducer

Browse Source
This commit is contained in:
Raven Scott 2023-08-21 21:15:10 -04:00
parent a980f3cd43
commit 1f3ff44317
1 changed files with 6 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
llamabot.js
View File

@ -454,7 +454,7 @@ async function generateResponse(conversation, message) {
// Remove older conversations if necessary
const maxLength = 1800;
const tolerance = 100; // allow for some flexibility in the token length
const tolerance = 25; // allow for some flexibility in the token length
if (tokenLength > maxLength + tolerance) {
const diff = tokenLength - (maxLength + tolerance);
let removedTokens = 0;
@ -462,22 +462,19 @@ async function generateResponse(conversation, message) {
// Iterate over the messages in reverse order
for (let i = messagesCopy.length - 1; i >= 0; i--) {
const message = messagesCopy[i];
const messageTokens = countLlamaTokens(message);
// Calculate the token length of the current message
const messageTokenLength = countLlamaTokens(messageTokens);
const messageTokens = countLlamaTokens([message]);
// Check if the current message plus the tokens in the message is less than or equal to the diff
if (removedTokens + messageTokenLength <= diff) {
if (removedTokens + messageTokens <= diff) {
messagesCopy.splice(i, 1);
removedTokens += messageTokenLength;
removedTokens += messageTokens;
console.log(removedTokens + " removed \nAfter Resize: " + countLlamaTokens(messagesCopy));
} else {
// Remove more than one message if necessary to bring the total length below the maximum allowed length
const messagesToRemove = Math.floor(diff / messageTokenLength);
const messagesToRemove = Math.floor(diff / messageTokens);
for (let j = 0; j < messagesToRemove; j++) {
messagesCopy.splice(i, 1);
removedTokens += messageTokenLength;
removedTokens += messageTokens;
}
break;
}
@ -510,7 +507,6 @@ async function generateResponse(conversation, message) {
if (time > 2) {
await botMessage.delete()
clearInterval(refreshInterval);
clearTimeout(timeout);
botMessage = null;
}