chore: format backend server code

This commit is contained in:
James Murdza 2024-11-16 21:35:06 -05:00
parent 87d311a5d1
commit 7ecbd02fef
8 changed files with 448 additions and 400 deletions

View File

@ -41,7 +41,11 @@ export class ConnectionManager {
} }
// Removes a connection for a sandbox // Removes a connection for a sandbox
removeConnectionForSandbox(socket: Socket, sandboxId: string, isOwner: boolean) { removeConnectionForSandbox(
socket: Socket,
sandboxId: string,
isOwner: boolean
) {
this.sockets[sandboxId]?.delete(socket) this.sockets[sandboxId]?.delete(socket)
// If the connection being removed is for the owner, decrements the owner connection counter // If the connection being removed is for the owner, decrements the owner connection counter
@ -52,7 +56,6 @@ export class ConnectionManager {
// Returns the set of sockets connected to a given sandbox // Returns the set of sockets connected to a given sandbox
connectionsForSandbox(sandboxId: string): Set<Socket> { connectionsForSandbox(sandboxId: string): Set<Socket> {
return this.sockets[sandboxId] ?? new Set(); return this.sockets[sandboxId] ?? new Set()
} }
} }

View File

@ -23,7 +23,11 @@ function generateFileStructure(paths: string[]): (TFolder | TFile)[] {
} }
} else { } else {
if (isFile) { if (isFile) {
const file: TFile = { id: `/${parts.join("/")}`, type: "file", name: part } const file: TFile = {
id: `/${parts.join("/")}`,
type: "file",
name: part,
}
current.children.push(file) current.children.push(file)
} else { } else {
const folder: TFolder = { const folder: TFolder = {
@ -75,7 +79,9 @@ export class FileManager {
if (isFile) { if (isFile) {
const fileId = `/${parts.join("/")}` const fileId = `/${parts.join("/")}`
const data = await RemoteFileStorage.fetchFileContent(`projects/${this.sandboxId}${fileId}`) const data = await RemoteFileStorage.fetchFileContent(
`projects/${this.sandboxId}${fileId}`
)
fileData.push({ id: fileId, data }) fileData.push({ id: fileId, data })
} }
} }
@ -91,7 +97,7 @@ export class FileManager {
// Convert remote file path to local file path // Convert remote file path to local file path
private getLocalFileId(remoteId: string): string | undefined { private getLocalFileId(remoteId: string): string | undefined {
const allParts = remoteId.split("/") const allParts = remoteId.split("/")
if (allParts[1] !== this.sandboxId) return undefined; if (allParts[1] !== this.sandboxId) return undefined
return allParts.slice(2).join("/") return allParts.slice(2).join("/")
} }
@ -99,7 +105,7 @@ export class FileManager {
private getLocalFileIds(remoteIds: string[]): string[] { private getLocalFileIds(remoteIds: string[]): string[] {
return remoteIds return remoteIds
.map(this.getLocalFileId.bind(this)) .map(this.getLocalFileId.bind(this))
.filter((id) => id !== undefined); .filter((id) => id !== undefined)
} }
// Download files from remote storage // Download files from remote storage
@ -120,7 +126,6 @@ export class FileManager {
// Initialize the FileManager // Initialize the FileManager
async initialize() { async initialize() {
// Download files from remote file storage // Download files from remote file storage
await this.updateFileStructure() await this.updateFileStructure()
await this.updateFileData() await this.updateFileData()
@ -141,10 +146,14 @@ export class FileManager {
await Promise.all(promises) await Promise.all(promises)
// Reload file list from the container to include template files // Reload file list from the container to include template files
const result = await this.sandbox.commands.run(`find "${this.dirName}" -type f`); // List all files recursively const result = await this.sandbox.commands.run(
const localPaths = result.stdout.split('\n').filter(path => path); // Split the output into an array and filter out empty strings `find "${this.dirName}" -type f`
const relativePaths = localPaths.map(filePath => path.posix.relative(this.dirName, filePath)); // Convert absolute paths to relative paths ) // List all files recursively
this.files = generateFileStructure(relativePaths); const localPaths = result.stdout.split("\n").filter((path) => path) // Split the output into an array and filter out empty strings
const relativePaths = localPaths.map((filePath) =>
path.posix.relative(this.dirName, filePath)
) // Convert absolute paths to relative paths
this.files = generateFileStructure(relativePaths)
// Make the logged in user the owner of all project files // Make the logged in user the owner of all project files
this.fixPermissions() this.fixPermissions()
@ -169,9 +178,7 @@ export class FileManager {
// Change the owner of the project directory to user // Change the owner of the project directory to user
private async fixPermissions() { private async fixPermissions() {
try { try {
await this.sandbox.commands.run( await this.sandbox.commands.run(`sudo chown -R user "${this.dirName}"`)
`sudo chown -R user "${this.dirName}"`
)
} catch (e: any) { } catch (e: any) {
console.log("Failed to fix permissions: " + e) console.log("Failed to fix permissions: " + e)
} }
@ -193,7 +200,10 @@ export class FileManager {
// This is the absolute file path in the container // This is the absolute file path in the container
const containerFilePath = path.posix.join(directory, event.name) const containerFilePath = path.posix.join(directory, event.name)
// This is the file path relative to the project directory // This is the file path relative to the project directory
const sandboxFilePath = removeDirName(containerFilePath, this.dirName) const sandboxFilePath = removeDirName(
containerFilePath,
this.dirName
)
// This is the directory being watched relative to the project directory // This is the directory being watched relative to the project directory
const sandboxDirectory = removeDirName(directory, this.dirName) const sandboxDirectory = removeDirName(directory, this.dirName)
@ -361,7 +371,9 @@ export class FileManager {
// Get folder content // Get folder content
async getFolder(folderId: string): Promise<string[]> { async getFolder(folderId: string): Promise<string[]> {
const remotePaths = await RemoteFileStorage.getFolder(this.getRemoteFileId(folderId)) const remotePaths = await RemoteFileStorage.getFolder(
this.getRemoteFileId(folderId)
)
return this.getLocalFileIds(remotePaths) return this.getLocalFileIds(remotePaths)
} }
@ -400,7 +412,11 @@ export class FileManager {
fileData.id = newFileId fileData.id = newFileId
file.id = newFileId file.id = newFileId
await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data) await RemoteFileStorage.renameFile(
this.getRemoteFileId(fileId),
this.getRemoteFileId(newFileId),
fileData.data
)
return this.updateFileStructure() return this.updateFileStructure()
} }
@ -465,7 +481,11 @@ export class FileManager {
await this.moveFileInContainer(fileId, newFileId) await this.moveFileInContainer(fileId, newFileId)
await this.fixPermissions() await this.fixPermissions()
await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data) await RemoteFileStorage.renameFile(
this.getRemoteFileId(fileId),
this.getRemoteFileId(newFileId),
fileData.data
)
fileData.id = newFileId fileData.id = newFileId
file.id = newFileId file.id = newFileId
@ -477,9 +497,7 @@ export class FileManager {
if (!file) return this.files if (!file) return this.files
await this.sandbox.files.remove(path.posix.join(this.dirName, fileId)) await this.sandbox.files.remove(path.posix.join(this.dirName, fileId))
this.fileData = this.fileData.filter( this.fileData = this.fileData.filter((f) => f.id !== fileId)
(f) => f.id !== fileId
)
await RemoteFileStorage.deleteFile(this.getRemoteFileId(fileId)) await RemoteFileStorage.deleteFile(this.getRemoteFileId(fileId))
return this.updateFileStructure() return this.updateFileStructure()
@ -487,14 +505,14 @@ export class FileManager {
// Delete a folder // Delete a folder
async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> { async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> {
const files = await RemoteFileStorage.getFolder(this.getRemoteFileId(folderId)) const files = await RemoteFileStorage.getFolder(
this.getRemoteFileId(folderId)
)
await Promise.all( await Promise.all(
files.map(async (file) => { files.map(async (file) => {
await this.sandbox.files.remove(path.posix.join(this.dirName, file)) await this.sandbox.files.remove(path.posix.join(this.dirName, file))
this.fileData = this.fileData.filter( this.fileData = this.fileData.filter((f) => f.id !== file)
(f) => f.id !== file
)
await RemoteFileStorage.deleteFile(this.getRemoteFileId(file)) await RemoteFileStorage.deleteFile(this.getRemoteFileId(file))
}) })
) )

View File

@ -61,11 +61,7 @@ export const RemoteFileStorage = {
return res.ok return res.ok
}, },
renameFile: async ( renameFile: async (fileId: string, newFileId: string, data: string) => {
fileId: string,
newFileId: string,
data: string
) => {
const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, {
method: "POST", method: "POST",
headers: { headers: {
@ -111,7 +107,7 @@ export const RemoteFileStorage = {
} }
) )
return (await res.json()).size return (await res.json()).size
} },
} }
export default RemoteFileStorage export default RemoteFileStorage

View File

@ -18,7 +18,7 @@ import { LockManager } from "./utils"
const lockManager = new LockManager() const lockManager = new LockManager()
// Define a type for SocketHandler functions // Define a type for SocketHandler functions
type SocketHandler<T = Record<string, any>> = (args: T) => any; type SocketHandler<T = Record<string, any>> = (args: T) => any
// Extract port number from a string // Extract port number from a string
function extractPortNumber(inputString: string): number | null { function extractPortNumber(inputString: string): number | null {
@ -29,34 +29,38 @@ function extractPortNumber(inputString: string): number | null {
} }
type ServerContext = { type ServerContext = {
aiWorker: AIWorker; aiWorker: AIWorker
dokkuClient: DokkuClient | null; dokkuClient: DokkuClient | null
gitClient: SecureGitClient | null; gitClient: SecureGitClient | null
}; }
export class Sandbox { export class Sandbox {
// Sandbox properties: // Sandbox properties:
sandboxId: string; sandboxId: string
type: string; type: string
fileManager: FileManager | null; fileManager: FileManager | null
terminalManager: TerminalManager | null; terminalManager: TerminalManager | null
container: E2BSandbox | null; container: E2BSandbox | null
// Server context: // Server context:
dokkuClient: DokkuClient | null; dokkuClient: DokkuClient | null
gitClient: SecureGitClient | null; gitClient: SecureGitClient | null
aiWorker: AIWorker; aiWorker: AIWorker
constructor(sandboxId: string, type: string, { aiWorker, dokkuClient, gitClient }: ServerContext) { constructor(
sandboxId: string,
type: string,
{ aiWorker, dokkuClient, gitClient }: ServerContext
) {
// Sandbox properties: // Sandbox properties:
this.sandboxId = sandboxId; this.sandboxId = sandboxId
this.type = type; this.type = type
this.fileManager = null; this.fileManager = null
this.terminalManager = null; this.terminalManager = null
this.container = null; this.container = null
// Server context: // Server context:
this.aiWorker = aiWorker; this.aiWorker = aiWorker
this.dokkuClient = dokkuClient; this.dokkuClient = dokkuClient
this.gitClient = gitClient; this.gitClient = gitClient
} }
// Initializes the container for the sandbox environment // Initializes the container for the sandbox environment
@ -66,15 +70,15 @@ export class Sandbox {
// Acquire a lock to ensure exclusive access to the sandbox environment // Acquire a lock to ensure exclusive access to the sandbox environment
await lockManager.acquireLock(this.sandboxId, async () => { await lockManager.acquireLock(this.sandboxId, async () => {
// Check if a container already exists and is running // Check if a container already exists and is running
if (this.container && await this.container.isRunning()) { if (this.container && (await this.container.isRunning())) {
console.log(`Found existing container ${this.sandboxId}`) console.log(`Found existing container ${this.sandboxId}`)
} else { } else {
console.log("Creating container", this.sandboxId) console.log("Creating container", this.sandboxId)
// Create a new container with a specified template and timeout // Create a new container with a specified template and timeout
const templateTypes = ["vanillajs", "reactjs", "nextjs", "streamlit"]; const templateTypes = ["vanillajs", "reactjs", "nextjs", "streamlit"]
const template = templateTypes.includes(this.type) const template = templateTypes.includes(this.type)
? `gitwit-${this.type}` ? `gitwit-${this.type}`
: `base`; : `base`
this.container = await E2BSandbox.create(template, { this.container = await E2BSandbox.create(template, {
timeoutMs: CONTAINER_TIMEOUT, timeoutMs: CONTAINER_TIMEOUT,
}) })
@ -106,15 +110,14 @@ export class Sandbox {
// Close all terminals managed by the terminal manager // Close all terminals managed by the terminal manager
await this.terminalManager?.closeAllTerminals() await this.terminalManager?.closeAllTerminals()
// This way the terminal manager will be set up again if we reconnect // This way the terminal manager will be set up again if we reconnect
this.terminalManager = null; this.terminalManager = null
// Close all file watchers managed by the file manager // Close all file watchers managed by the file manager
await this.fileManager?.closeWatchers() await this.fileManager?.closeWatchers()
// This way the file manager will be set up again if we reconnect // This way the file manager will be set up again if we reconnect
this.fileManager = null; this.fileManager = null
} }
handlers(connection: { userId: string, isOwner: boolean, socket: Socket }) { handlers(connection: { userId: string; isOwner: boolean; socket: Socket }) {
// Handle heartbeat from a socket connection // Handle heartbeat from a socket connection
const handleHeartbeat: SocketHandler = (_: any) => { const handleHeartbeat: SocketHandler = (_: any) => {
// Only keep the sandbox alive if the owner is still connected // Only keep the sandbox alive if the owner is still connected
@ -135,7 +138,7 @@ export class Sandbox {
// Handle saving a file // Handle saving a file
const handleSaveFile: SocketHandler = async ({ fileId, body }: any) => { const handleSaveFile: SocketHandler = async ({ fileId, body }: any) => {
await saveFileRL.consume(connection.userId, 1); await saveFileRL.consume(connection.userId, 1)
return this.fileManager?.saveFile(fileId, body) return this.fileManager?.saveFile(fileId, body)
} }
@ -146,7 +149,8 @@ export class Sandbox {
// Handle listing apps // Handle listing apps
const handleListApps: SocketHandler = async (_: any) => { const handleListApps: SocketHandler = async (_: any) => {
if (!this.dokkuClient) throw Error("Failed to retrieve apps list: No Dokku client") if (!this.dokkuClient)
throw Error("Failed to retrieve apps list: No Dokku client")
return { success: true, apps: await this.dokkuClient.listApps() } return { success: true, apps: await this.dokkuClient.listApps() }
} }
@ -160,18 +164,21 @@ export class Sandbox {
// Handle creating a file // Handle creating a file
const handleCreateFile: SocketHandler = async ({ name }: any) => { const handleCreateFile: SocketHandler = async ({ name }: any) => {
await createFileRL.consume(connection.userId, 1); await createFileRL.consume(connection.userId, 1)
return { "success": await this.fileManager?.createFile(name) } return { success: await this.fileManager?.createFile(name) }
} }
// Handle creating a folder // Handle creating a folder
const handleCreateFolder: SocketHandler = async ({ name }: any) => { const handleCreateFolder: SocketHandler = async ({ name }: any) => {
await createFolderRL.consume(connection.userId, 1); await createFolderRL.consume(connection.userId, 1)
return { "success": await this.fileManager?.createFolder(name) } return { success: await this.fileManager?.createFolder(name) }
} }
// Handle renaming a file // Handle renaming a file
const handleRenameFile: SocketHandler = async ({ fileId, newName }: any) => { const handleRenameFile: SocketHandler = async ({
fileId,
newName,
}: any) => {
await renameFileRL.consume(connection.userId, 1) await renameFileRL.consume(connection.userId, 1)
return this.fileManager?.renameFile(fileId, newName) return this.fileManager?.renameFile(fileId, newName)
} }
@ -190,8 +197,13 @@ export class Sandbox {
// Handle creating a terminal session // Handle creating a terminal session
const handleCreateTerminal: SocketHandler = async ({ id }: any) => { const handleCreateTerminal: SocketHandler = async ({ id }: any) => {
await lockManager.acquireLock(this.sandboxId, async () => { await lockManager.acquireLock(this.sandboxId, async () => {
await this.terminalManager?.createTerminal(id, (responseString: string) => { await this.terminalManager?.createTerminal(
connection.socket.emit("terminalResponse", { id, data: responseString }) id,
(responseString: string) => {
connection.socket.emit("terminalResponse", {
id,
data: responseString,
})
const port = extractPortNumber(responseString) const port = extractPortNumber(responseString)
if (port) { if (port) {
connection.socket.emit( connection.socket.emit(
@ -199,7 +211,8 @@ export class Sandbox {
"https://" + this.container?.getHost(port) "https://" + this.container?.getHost(port)
) )
} }
}) }
)
}) })
} }
@ -219,8 +232,19 @@ export class Sandbox {
} }
// Handle generating code // Handle generating code
const handleGenerateCode: SocketHandler = ({ fileName, code, line, instructions }: any) => { const handleGenerateCode: SocketHandler = ({
return this.aiWorker.generateCode(connection.userId, fileName, code, line, instructions) fileName,
code,
line,
instructions,
}: any) => {
return this.aiWorker.generateCode(
connection.userId,
fileName,
code,
line,
instructions
)
} }
// Handle downloading files by download button // Handle downloading files by download button
@ -229,34 +253,32 @@ export class Sandbox {
// Get all files with their data through fileManager // Get all files with their data through fileManager
const files = this.fileManager.fileData.map((file: TFileData) => ({ const files = this.fileManager.fileData.map((file: TFileData) => ({
path: file.id.startsWith('/') ? file.id.slice(1) : file.id, path: file.id.startsWith("/") ? file.id.slice(1) : file.id,
content: file.data content: file.data,
})) }))
return { files } return { files }
} }
return { return {
"heartbeat": handleHeartbeat, heartbeat: handleHeartbeat,
"getFile": handleGetFile, getFile: handleGetFile,
"downloadFiles": handleDownloadFiles, downloadFiles: handleDownloadFiles,
"getFolder": handleGetFolder, getFolder: handleGetFolder,
"saveFile": handleSaveFile, saveFile: handleSaveFile,
"moveFile": handleMoveFile, moveFile: handleMoveFile,
"list": handleListApps, list: handleListApps,
"deploy": handleDeploy, deploy: handleDeploy,
"createFile": handleCreateFile, createFile: handleCreateFile,
"createFolder": handleCreateFolder, createFolder: handleCreateFolder,
"renameFile": handleRenameFile, renameFile: handleRenameFile,
"deleteFile": handleDeleteFile, deleteFile: handleDeleteFile,
"deleteFolder": handleDeleteFolder, deleteFolder: handleDeleteFolder,
"createTerminal": handleCreateTerminal, createTerminal: handleCreateTerminal,
"resizeTerminal": handleResizeTerminal, resizeTerminal: handleResizeTerminal,
"terminalData": handleTerminalData, terminalData: handleTerminalData,
"closeTerminal": handleCloseTerminal, closeTerminal: handleCloseTerminal,
"generateCode": handleGenerateCode, generateCode: handleGenerateCode,
}; }
} }
} }

View File

@ -10,14 +10,14 @@ import { ConnectionManager } from "./ConnectionManager"
import { DokkuClient } from "./DokkuClient" import { DokkuClient } from "./DokkuClient"
import { Sandbox } from "./Sandbox" import { Sandbox } from "./Sandbox"
import { SecureGitClient } from "./SecureGitClient" import { SecureGitClient } from "./SecureGitClient"
import { socketAuth } from "./socketAuth"; // Import the new socketAuth middleware import { socketAuth } from "./socketAuth" // Import the new socketAuth middleware
import { TFile, TFolder } from "./types" import { TFile, TFolder } from "./types"
// Log errors and send a notification to the client // Log errors and send a notification to the client
export const handleErrors = (message: string, error: any, socket: Socket) => { export const handleErrors = (message: string, error: any, socket: Socket) => {
console.error(message, error); console.error(message, error)
socket.emit("error", `${message} ${error.message ?? error}`); socket.emit("error", `${message} ${error.message ?? error}`)
}; }
// Handle uncaught exceptions // Handle uncaught exceptions
process.on("uncaughtException", (error) => { process.on("uncaughtException", (error) => {
@ -110,21 +110,23 @@ io.on("connection", async (socket) => {
try { try {
// Create or retrieve the sandbox manager for the given sandbox ID // Create or retrieve the sandbox manager for the given sandbox ID
const sandbox = sandboxes[data.sandboxId] ?? new Sandbox( const sandbox =
data.sandboxId, sandboxes[data.sandboxId] ??
data.type, new Sandbox(data.sandboxId, data.type, {
{ aiWorker,
aiWorker, dokkuClient, gitClient, dokkuClient,
} gitClient,
) })
sandboxes[data.sandboxId] = sandbox sandboxes[data.sandboxId] = sandbox
// This callback recieves an update when the file list changes, and notifies all relevant connections. // This callback recieves an update when the file list changes, and notifies all relevant connections.
const sendFileNotifications = (files: (TFolder | TFile)[]) => { const sendFileNotifications = (files: (TFolder | TFile)[]) => {
connections.connectionsForSandbox(data.sandboxId).forEach((socket: Socket) => { connections
socket.emit("loaded", files); .connectionsForSandbox(data.sandboxId)
}); .forEach((socket: Socket) => {
}; socket.emit("loaded", files)
})
}
// Initialize the sandbox container // Initialize the sandbox container
// The file manager and terminal managers will be set up if they have been closed // The file manager and terminal managers will be set up if they have been closed
@ -134,26 +136,35 @@ io.on("connection", async (socket) => {
// Register event handlers for the sandbox // Register event handlers for the sandbox
// For each event handler, listen on the socket for that event // For each event handler, listen on the socket for that event
// Pass connection-specific information to the handlers // Pass connection-specific information to the handlers
Object.entries(sandbox.handlers({ Object.entries(
sandbox.handlers({
userId: data.userId, userId: data.userId,
isOwner: data.isOwner, isOwner: data.isOwner,
socket socket,
})).forEach(([event, handler]) => { })
socket.on(event, async (options: any, callback?: (response: any) => void) => { ).forEach(([event, handler]) => {
socket.on(
event,
async (options: any, callback?: (response: any) => void) => {
try { try {
const result = await handler(options) const result = await handler(options)
callback?.(result); callback?.(result)
} catch (e: any) { } catch (e: any) {
handleErrors(`Error processing event "${event}":`, e, socket); handleErrors(`Error processing event "${event}":`, e, socket)
} }
}); }
}); )
})
// Handle disconnection event // Handle disconnection event
socket.on("disconnect", async () => { socket.on("disconnect", async () => {
try { try {
// Deregister the connection // Deregister the connection
connections.removeConnectionForSandbox(socket, data.sandboxId, data.isOwner) connections.removeConnectionForSandbox(
socket,
data.sandboxId,
data.isOwner
)
// If the owner has disconnected from all sockets, close open terminals and file watchers.o // If the owner has disconnected from all sockets, close open terminals and file watchers.o
// The sandbox itself will timeout after the heartbeat stops. // The sandbox itself will timeout after the heartbeat stops.
@ -165,16 +176,14 @@ io.on("connection", async (socket) => {
) )
} }
} catch (e: any) { } catch (e: any) {
handleErrors("Error disconnecting:", e, socket); handleErrors("Error disconnecting:", e, socket)
} }
}) })
} catch (e: any) { } catch (e: any) {
handleErrors(`Error initializing sandbox ${data.sandboxId}:`, e, socket); handleErrors(`Error initializing sandbox ${data.sandboxId}:`, e, socket)
} }
} catch (e: any) { } catch (e: any) {
handleErrors("Error connecting:", e, socket); handleErrors("Error connecting:", e, socket)
} }
}) })

View File

@ -67,7 +67,7 @@ export const socketAuth = async (socket: Socket, next: Function) => {
userId, userId,
sandboxId: sandboxId, sandboxId: sandboxId,
isOwner: sandbox !== undefined, isOwner: sandbox !== undefined,
type: dbSandboxJSON.type type: dbSandboxJSON.type,
} }
// Allow the connection // Allow the connection