From da5e2e6e7dde2c4ec9b24073fadcbf3db5bd8937 Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Thu, 27 Apr 2023 02:56:27 +0200 Subject: [PATCH] adding N_THREADS and MAX_TOKENS vars --- default.env | 2 ++ docker-compose.yml | 2 +- llamabot.js | 9 +++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/default.env b/default.env index 6e7aa55..6723905 100644 --- a/default.env +++ b/default.env @@ -4,6 +4,8 @@ INIT_PROMPT = "Assistant name: ChatAI. You code, write and provide any informati ROOT_PORT = 8000 DATA_DIR = /home/USERNAME/weights CACHE = 1 +N_THREADS = 4 +MAX_TOKENS = 1024 # ROOT_IP is only used when running the bot without docker compose ROOT_IP = 192.168.0.15 diff --git a/docker-compose.yml b/docker-compose.yml index 151f3dc..e6cf2ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,4 +24,4 @@ services: - CHANNEL_IDS - ROOT_IP=llama-python-server - ROOT_PORT=8000 - - INIT_PROMPT='Assistant name':' ChatAI. You code, write and provide any information without any mistakes.' + - INIT_PROMPT='Assistant Name':' ChatAI. You code, write and provide any information without any mistakes.' diff --git a/llamabot.js b/llamabot.js index 54d9c99..159ccf3 100644 --- a/llamabot.js +++ b/llamabot.js @@ -231,14 +231,15 @@ async function generateResponse(conversation, message) { const response = await fetch(`http://${process.env.ROOT_IP}:${process.env.ROOT_PORT}/v1/chat/completions`, { method: 'POST', headers: { - 'accept': 'application/json', - 'Content-Type': 'application/json' + 'accept': 'application/json', + 'Content-Type': 'application/json' }, body: JSON.stringify({ - messages: messagesCopy // use the copy of the messages array + messages: messagesCopy, + max_tokens: process.env.MAX_TOKENS // add the max_tokens parameter here }), signal: controller.signal - }); + }); const responseData = await response.json(); console.log(JSON.stringify(responseData));