From 1416c225a28ff30b441ed72412bda7d6b83c6359 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 05:22:42 -0600 Subject: [PATCH 01/10] chore: add code formatting settings --- .prettierignore | 4 ++++ .vscode/settings.json | 8 ++++++++ backend/server/.prettierrc | 6 ++++++ 3 files changed, 18 insertions(+) create mode 100644 .prettierignore create mode 100644 .vscode/settings.json create mode 100644 backend/server/.prettierrc diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..4314ec3 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,4 @@ +frontend/** +backend/ai/** +backend/database/** +backend/storage/** \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4b81cce --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "file", + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + "source.organizeImports": "explicit" + } +} diff --git a/backend/server/.prettierrc b/backend/server/.prettierrc new file mode 100644 index 0000000..c2e595e --- /dev/null +++ b/backend/server/.prettierrc @@ -0,0 +1,6 @@ +{ + "tabWidth": 2, + "semi": false, + "singleQuote": false, + "insertFinalNewline": true +} \ No newline at end of file From ad9457b157b8ab5629aee09d7c2cc5b89ce4a68a Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 05:25:26 -0600 Subject: [PATCH 02/10] chore: format backend server code --- backend/server/nodemon.json | 4 +- backend/server/package.json | 2 +- backend/server/src/DokkuClient.ts | 26 +- backend/server/src/SSHSocketClient.ts | 168 ++--- backend/server/src/SecureGitClient.ts | 80 +-- backend/server/src/Terminal.ts | 35 +- backend/server/src/fileoperations.ts | 117 ++-- backend/server/src/index.ts | 880 ++++++++++++++------------ backend/server/src/ratelimit.ts | 2 +- backend/server/src/types.ts | 102 +-- backend/server/src/utils.ts | 18 +- 11 files changed, 765 insertions(+), 669 deletions(-) diff --git a/backend/server/nodemon.json b/backend/server/nodemon.json index 5554d0f..c71b99b 100644 --- a/backend/server/nodemon.json +++ b/backend/server/nodemon.json @@ -1,5 +1,7 @@ { - "watch": ["src"], + "watch": [ + "src" + ], "ext": "ts", "exec": "concurrently \"npx tsc --watch\" \"ts-node src/index.ts\"" } \ No newline at end of file diff --git a/backend/server/package.json b/backend/server/package.json index 40c9c18..435cd1b 100644 --- a/backend/server/package.json +++ b/backend/server/package.json @@ -31,4 +31,4 @@ "ts-node": "^10.9.2", "typescript": "^5.4.5" } -} +} \ No newline at end of file diff --git a/backend/server/src/DokkuClient.ts b/backend/server/src/DokkuClient.ts index fd0adcd..e2e9911 100644 --- a/backend/server/src/DokkuClient.ts +++ b/backend/server/src/DokkuClient.ts @@ -1,37 +1,33 @@ -import { SSHSocketClient, SSHConfig } from "./SSHSocketClient" +import { SSHConfig, SSHSocketClient } from "./SSHSocketClient" export interface DokkuResponse { - ok: boolean; - output: string; + ok: boolean + output: string } export class DokkuClient extends SSHSocketClient { - constructor(config: SSHConfig) { - super( - config, - "/var/run/dokku-daemon/dokku-daemon.sock" - ) + super(config, "/var/run/dokku-daemon/dokku-daemon.sock") } async sendCommand(command: string): Promise { try { - const response = await this.sendData(command); + const response = await this.sendData(command) if (typeof response !== "string") { - throw new Error("Received data is not a string"); + throw new Error("Received data is not a string") } - return JSON.parse(response); + return JSON.parse(response) } catch (error: any) { - throw new Error(`Failed to send command: ${error.message}`); + throw new Error(`Failed to send command: ${error.message}`) } } async listApps(): Promise { - const response = await this.sendCommand("apps:list"); - return response.output.split("\n").slice(1); // Split by newline and ignore the first line (header) + const response = await this.sendCommand("apps:list") + return response.output.split("\n").slice(1) // Split by newline and ignore the first line (header) } } -export { SSHConfig }; \ No newline at end of file +export { SSHConfig } diff --git a/backend/server/src/SSHSocketClient.ts b/backend/server/src/SSHSocketClient.ts index e0dc043..c653b23 100644 --- a/backend/server/src/SSHSocketClient.ts +++ b/backend/server/src/SSHSocketClient.ts @@ -1,90 +1,90 @@ -import { Client } from "ssh2"; +import { Client } from "ssh2" export interface SSHConfig { - host: string; - port?: number; - username: string; - privateKey: Buffer; + host: string + port?: number + username: string + privateKey: Buffer } export class SSHSocketClient { - private conn: Client; - private config: SSHConfig; - private socketPath: string; - private isConnected: boolean = false; - - constructor(config: SSHConfig, socketPath: string) { - this.conn = new Client(); - this.config = { ...config, port: 22}; - this.socketPath = socketPath; - - this.setupTerminationHandlers(); - } - - private setupTerminationHandlers() { - process.on("SIGINT", this.closeConnection.bind(this)); - process.on("SIGTERM", this.closeConnection.bind(this)); - } - - private closeConnection() { - console.log("Closing SSH connection..."); - this.conn.end(); - this.isConnected = false; - process.exit(0); - } - - connect(): Promise { - return new Promise((resolve, reject) => { - this.conn - .on("ready", () => { - console.log("SSH connection established"); - this.isConnected = true; - resolve(); - }) - .on("error", (err) => { - console.error("SSH connection error:", err); - this.isConnected = false; - reject(err); - }) - .on("close", () => { - console.log("SSH connection closed"); - this.isConnected = false; - }) - .connect(this.config); - }); - } - - sendData(data: string): Promise { - return new Promise((resolve, reject) => { - if (!this.isConnected) { - reject(new Error("SSH connection is not established")); - return; - } - - this.conn.exec( - `echo "${data}" | nc -U ${this.socketPath}`, - (err, stream) => { - if (err) { - reject(err); - return; - } - - stream - .on("close", (code: number, signal: string) => { - reject( - new Error( - `Stream closed with code ${code} and signal ${signal}` - ) - ); - }) - .on("data", (data: Buffer) => { - resolve(data.toString()); - }) - .stderr.on("data", (data: Buffer) => { - reject(new Error(data.toString())); - }); + private conn: Client + private config: SSHConfig + private socketPath: string + private isConnected: boolean = false + + constructor(config: SSHConfig, socketPath: string) { + this.conn = new Client() + this.config = { ...config, port: 22 } + this.socketPath = socketPath + + this.setupTerminationHandlers() + } + + private setupTerminationHandlers() { + process.on("SIGINT", this.closeConnection.bind(this)) + process.on("SIGTERM", this.closeConnection.bind(this)) + } + + private closeConnection() { + console.log("Closing SSH connection...") + this.conn.end() + this.isConnected = false + process.exit(0) + } + + connect(): Promise { + return new Promise((resolve, reject) => { + this.conn + .on("ready", () => { + console.log("SSH connection established") + this.isConnected = true + resolve() + }) + .on("error", (err) => { + console.error("SSH connection error:", err) + this.isConnected = false + reject(err) + }) + .on("close", () => { + console.log("SSH connection closed") + this.isConnected = false + }) + .connect(this.config) + }) + } + + sendData(data: string): Promise { + return new Promise((resolve, reject) => { + if (!this.isConnected) { + reject(new Error("SSH connection is not established")) + return + } + + this.conn.exec( + `echo "${data}" | nc -U ${this.socketPath}`, + (err, stream) => { + if (err) { + reject(err) + return } - ); - }); - } - } \ No newline at end of file + + stream + .on("close", (code: number, signal: string) => { + reject( + new Error( + `Stream closed with code ${code} and signal ${signal}` + ) + ) + }) + .on("data", (data: Buffer) => { + resolve(data.toString()) + }) + .stderr.on("data", (data: Buffer) => { + reject(new Error(data.toString())) + }) + } + ) + }) + } +} diff --git a/backend/server/src/SecureGitClient.ts b/backend/server/src/SecureGitClient.ts index 6fabce6..34f5322 100644 --- a/backend/server/src/SecureGitClient.ts +++ b/backend/server/src/SecureGitClient.ts @@ -1,82 +1,84 @@ -import simpleGit, { SimpleGit } from "simple-git"; -import path from "path"; -import fs from "fs"; -import os from "os"; +import fs from "fs" +import os from "os" +import path from "path" +import simpleGit, { SimpleGit } from "simple-git" export type FileData = { - id: string; - data: string; -}; + id: string + data: string +} export class SecureGitClient { - private gitUrl: string; - private sshKeyPath: string; + private gitUrl: string + private sshKeyPath: string constructor(gitUrl: string, sshKeyPath: string) { - this.gitUrl = gitUrl; - this.sshKeyPath = sshKeyPath; + this.gitUrl = gitUrl + this.sshKeyPath = sshKeyPath } async pushFiles(fileData: FileData[], repository: string): Promise { - let tempDir: string | undefined; + let tempDir: string | undefined try { // Create a temporary directory - tempDir = fs.mkdtempSync(path.posix.join(os.tmpdir(), 'git-push-')); - console.log(`Temporary directory created: ${tempDir}`); + tempDir = fs.mkdtempSync(path.posix.join(os.tmpdir(), "git-push-")) + console.log(`Temporary directory created: ${tempDir}`) // Write files to the temporary directory - console.log(`Writing ${fileData.length} files.`); + console.log(`Writing ${fileData.length} files.`) for (const { id, data } of fileData) { - const filePath = path.posix.join(tempDir, id); - const dirPath = path.dirname(filePath); - + const filePath = path.posix.join(tempDir, id) + const dirPath = path.dirname(filePath) + if (!fs.existsSync(dirPath)) { - fs.mkdirSync(dirPath, { recursive: true }); + fs.mkdirSync(dirPath, { recursive: true }) } - fs.writeFileSync(filePath, data); + fs.writeFileSync(filePath, data) } // Initialize the simple-git instance with the temporary directory and custom SSH command const git: SimpleGit = simpleGit(tempDir, { config: [ - 'core.sshCommand=ssh -i ' + this.sshKeyPath + ' -o IdentitiesOnly=yes' - ] + "core.sshCommand=ssh -i " + + this.sshKeyPath + + " -o IdentitiesOnly=yes", + ], }).outputHandler((_command, stdout, stderr) => { - stdout.pipe(process.stdout); - stderr.pipe(process.stderr); - });; + stdout.pipe(process.stdout) + stderr.pipe(process.stderr) + }) // Initialize a new Git repository - await git.init(); + await git.init() // Add remote repository - await git.addRemote("origin", `${this.gitUrl}:${repository}`); + await git.addRemote("origin", `${this.gitUrl}:${repository}`) // Add files to the repository - for (const {id, data} of fileData) { - await git.add(id); + for (const { id, data } of fileData) { + await git.add(id) } // Commit the changes - await git.commit("Add files."); + await git.commit("Add files.") // Push the changes to the remote repository - await git.push("origin", "master", {'--force': null}); + await git.push("origin", "master", { "--force": null }) - console.log("Files successfully pushed to the repository"); + console.log("Files successfully pushed to the repository") if (tempDir) { - fs.rmSync(tempDir, { recursive: true, force: true }); - console.log(`Temporary directory removed: ${tempDir}`); + fs.rmSync(tempDir, { recursive: true, force: true }) + console.log(`Temporary directory removed: ${tempDir}`) } } catch (error) { if (tempDir) { - fs.rmSync(tempDir, { recursive: true, force: true }); - console.log(`Temporary directory removed: ${tempDir}`); + fs.rmSync(tempDir, { recursive: true, force: true }) + console.log(`Temporary directory removed: ${tempDir}`) } - console.error("Error pushing files to the repository:", error); - throw error; + console.error("Error pushing files to the repository:", error) + throw error } } -} \ No newline at end of file +} diff --git a/backend/server/src/Terminal.ts b/backend/server/src/Terminal.ts index e30f022..482b8a4 100644 --- a/backend/server/src/Terminal.ts +++ b/backend/server/src/Terminal.ts @@ -1,13 +1,13 @@ -import { Sandbox, ProcessHandle } from "e2b"; +import { ProcessHandle, Sandbox } from "e2b" // Terminal class to manage a pseudo-terminal (PTY) in a sandbox environment export class Terminal { - private pty: ProcessHandle | undefined; // Holds the PTY process handle - private sandbox: Sandbox; // Reference to the sandbox environment + private pty: ProcessHandle | undefined // Holds the PTY process handle + private sandbox: Sandbox // Reference to the sandbox environment // Constructor initializes the Terminal with a sandbox constructor(sandbox: Sandbox) { - this.sandbox = sandbox; + this.sandbox = sandbox } // Initialize the terminal with specified rows, columns, and data handler @@ -16,9 +16,9 @@ export class Terminal { cols = 80, onData, }: { - rows?: number; - cols?: number; - onData: (responseData: string) => void; + rows?: number + cols?: number + onData: (responseData: string) => void }): Promise { // Create a new PTY process this.pty = await this.sandbox.pty.create({ @@ -26,35 +26,38 @@ export class Terminal { cols, timeout: 0, onData: (data: Uint8Array) => { - onData(new TextDecoder().decode(data)); // Convert received data to string and pass to handler + onData(new TextDecoder().decode(data)) // Convert received data to string and pass to handler }, - }); + }) } // Send data to the terminal async sendData(data: string) { if (this.pty) { - await this.sandbox.pty.sendInput(this.pty.pid, new TextEncoder().encode(data)); + await this.sandbox.pty.sendInput( + this.pty.pid, + new TextEncoder().encode(data) + ) } else { - console.log("Cannot send data because pty is not initialized."); + console.log("Cannot send data because pty is not initialized.") } } // Resize the terminal async resize(size: { cols: number; rows: number }): Promise { if (this.pty) { - await this.sandbox.pty.resize(this.pty.pid, size); + await this.sandbox.pty.resize(this.pty.pid, size) } else { - console.log("Cannot resize terminal because pty is not initialized."); + console.log("Cannot resize terminal because pty is not initialized.") } } // Close the terminal, killing the PTY process and stopping the input stream async close(): Promise { if (this.pty) { - await this.pty.kill(); + await this.pty.kill() } else { - console.log("Cannot kill pty because it is not initialized."); + console.log("Cannot kill pty because it is not initialized.") } } } @@ -64,4 +67,4 @@ export class Terminal { // await terminal.init(); // terminal.sendData('ls -la'); // await terminal.resize({ cols: 100, rows: 30 }); -// await terminal.close(); \ No newline at end of file +// await terminal.close(); diff --git a/backend/server/src/fileoperations.ts b/backend/server/src/fileoperations.ts index 5e0e249..1157487 100644 --- a/backend/server/src/fileoperations.ts +++ b/backend/server/src/fileoperations.ts @@ -1,14 +1,7 @@ -import * as dotenv from "dotenv"; -import { - R2FileBody, - R2Files, - Sandbox, - TFile, - TFileData, - TFolder, -} from "./types"; +import * as dotenv from "dotenv" +import { R2Files, TFile, TFileData, TFolder } from "./types" -dotenv.config(); +dotenv.config() export const getSandboxFiles = async (id: string) => { const res = await fetch( @@ -18,13 +11,13 @@ export const getSandboxFiles = async (id: string) => { Authorization: `${process.env.WORKERS_KEY}`, }, } - ); - const data: R2Files = await res.json(); + ) + const data: R2Files = await res.json() - const paths = data.objects.map((obj) => obj.key); - const processedFiles = await processFiles(paths, id); - return processedFiles; -}; + const paths = data.objects.map((obj) => obj.key) + const processedFiles = await processFiles(paths, id) + return processedFiles +} export const getFolder = async (folderId: string) => { const res = await fetch( @@ -34,39 +27,39 @@ export const getFolder = async (folderId: string) => { Authorization: `${process.env.WORKERS_KEY}`, }, } - ); - const data: R2Files = await res.json(); + ) + const data: R2Files = await res.json() - return data.objects.map((obj) => obj.key); -}; + return data.objects.map((obj) => obj.key) +} const processFiles = async (paths: string[], id: string) => { - const root: TFolder = { id: "/", type: "folder", name: "/", children: [] }; - const fileData: TFileData[] = []; + const root: TFolder = { id: "/", type: "folder", name: "/", children: [] } + const fileData: TFileData[] = [] paths.forEach((path) => { - const allParts = path.split("/"); + const allParts = path.split("/") if (allParts[1] !== id) { - return; + return } - const parts = allParts.slice(2); - let current: TFolder = root; + const parts = allParts.slice(2) + let current: TFolder = root for (let i = 0; i < parts.length; i++) { - const part = parts[i]; - const isFile = i === parts.length - 1 && part.length; - const existing = current.children.find((child) => child.name === part); + const part = parts[i] + const isFile = i === parts.length - 1 && part.length + const existing = current.children.find((child) => child.name === part) if (existing) { if (!isFile) { - current = existing as TFolder; + current = existing as TFolder } } else { if (isFile) { - const file: TFile = { id: path, type: "file", name: part }; - current.children.push(file); - fileData.push({ id: path, data: "" }); + const file: TFile = { id: path, type: "file", name: part } + current.children.push(file) + fileData.push({ id: path, data: "" }) } else { const folder: TFolder = { // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css @@ -74,26 +67,26 @@ const processFiles = async (paths: string[], id: string) => { type: "folder", name: part, children: [], - }; - current.children.push(folder); - current = folder; + } + current.children.push(folder) + current = folder } } } - }); + }) await Promise.all( fileData.map(async (file) => { - const data = await fetchFileContent(file.id); - file.data = data; + const data = await fetchFileContent(file.id) + file.data = data }) - ); + ) return { files: root.children, fileData, - }; -}; + } +} const fetchFileContent = async (fileId: string): Promise => { try { @@ -104,13 +97,13 @@ const fetchFileContent = async (fileId: string): Promise => { Authorization: `${process.env.WORKERS_KEY}`, }, } - ); - return await fileRes.text(); + ) + return await fileRes.text() } catch (error) { - console.error("ERROR fetching file:", error); - return ""; + console.error("ERROR fetching file:", error) + return "" } -}; +} export const createFile = async (fileId: string) => { const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { @@ -120,9 +113,9 @@ export const createFile = async (fileId: string) => { Authorization: `${process.env.WORKERS_KEY}`, }, body: JSON.stringify({ fileId }), - }); - return res.ok; -}; + }) + return res.ok +} export const renameFile = async ( fileId: string, @@ -136,9 +129,9 @@ export const renameFile = async ( Authorization: `${process.env.WORKERS_KEY}`, }, body: JSON.stringify({ fileId, newFileId, data }), - }); - return res.ok; -}; + }) + return res.ok +} export const saveFile = async (fileId: string, data: string) => { const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { @@ -148,9 +141,9 @@ export const saveFile = async (fileId: string, data: string) => { Authorization: `${process.env.WORKERS_KEY}`, }, body: JSON.stringify({ fileId, data }), - }); - return res.ok; -}; + }) + return res.ok +} export const deleteFile = async (fileId: string) => { const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { @@ -160,9 +153,9 @@ export const deleteFile = async (fileId: string) => { Authorization: `${process.env.WORKERS_KEY}`, }, body: JSON.stringify({ fileId }), - }); - return res.ok; -}; + }) + return res.ok +} export const getProjectSize = async (id: string) => { const res = await fetch( @@ -172,6 +165,6 @@ export const getProjectSize = async (id: string) => { Authorization: `${process.env.WORKERS_KEY}`, }, } - ); - return (await res.json()).size; -}; \ No newline at end of file + ) + return (await res.json()).size +} diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index fb8bec1..f659c7f 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -1,20 +1,14 @@ -import path from "path"; -import cors from "cors"; -import express, { Express } from "express"; -import dotenv from "dotenv"; -import { createServer } from "http"; -import { Server } from "socket.io"; -import { DokkuClient } from "./DokkuClient"; -import { SecureGitClient, FileData } from "./SecureGitClient"; -import fs, { readFile } from "fs"; +import cors from "cors" +import dotenv from "dotenv" +import express, { Express } from "express" +import fs from "fs" +import { createServer } from "http" +import path from "path" +import { Server } from "socket.io" +import { DokkuClient } from "./DokkuClient" +import { SecureGitClient } from "./SecureGitClient" -import { z } from "zod"; -import { - TFile, - TFileData, - TFolder, - User -} from "./types"; +import { z } from "zod" import { createFile, deleteFile, @@ -23,10 +17,17 @@ import { getSandboxFiles, renameFile, saveFile, -} from "./fileoperations"; -import { LockManager } from "./utils"; +} from "./fileoperations" +import { TFile, TFileData, TFolder, User } from "./types" +import { LockManager } from "./utils" -import { Sandbox, Filesystem, FilesystemEvent, EntryInfo, WatchHandle } from "e2b"; +import { + EntryInfo, + Filesystem, + FilesystemEvent, + Sandbox, + WatchHandle, +} from "e2b" import { Terminal } from "./Terminal" @@ -37,53 +38,57 @@ import { deleteFileRL, renameFileRL, saveFileRL, -} from "./ratelimit"; +} from "./ratelimit" -process.on('uncaughtException', (error) => { - console.error('Uncaught Exception:', error); +process.on("uncaughtException", (error) => { + console.error("Uncaught Exception:", error) // Do not exit the process // You can add additional logging or recovery logic here -}); +}) -process.on('unhandledRejection', (reason, promise) => { - console.error('Unhandled Rejection at:', promise, 'reason:', reason); +process.on("unhandledRejection", (reason, promise) => { + console.error("Unhandled Rejection at:", promise, "reason:", reason) // Do not exit the process // You can also handle the rejected promise here if needed -}); +}) // The amount of time in ms that a container will stay alive without a hearbeat. -const CONTAINER_TIMEOUT = 60_000; +const CONTAINER_TIMEOUT = 60_000 -dotenv.config(); +dotenv.config() -const app: Express = express(); -const port = process.env.PORT || 4000; -app.use(cors()); -const httpServer = createServer(app); +const app: Express = express() +const port = process.env.PORT || 4000 +app.use(cors()) +const httpServer = createServer(app) const io = new Server(httpServer, { cors: { origin: "*", }, -}); +}) -let inactivityTimeout: NodeJS.Timeout | null = null; -let isOwnerConnected = false; +let inactivityTimeout: NodeJS.Timeout | null = null +let isOwnerConnected = false -const containers: Record = {}; -const connections: Record = {}; -const terminals: Record = {}; +const containers: Record = {} +const connections: Record = {} +const terminals: Record = {} -const dirName = "/home/user"; +const dirName = "/home/user" -const moveFile = async (filesystem: Filesystem, filePath: string, newFilePath: string) => { +const moveFile = async ( + filesystem: Filesystem, + filePath: string, + newFilePath: string +) => { try { - const fileContents = await filesystem.read(filePath); - await filesystem.write(newFilePath, fileContents); - await filesystem.remove(filePath); + const fileContents = await filesystem.read(filePath) + await filesystem.write(newFilePath, fileContents) + await filesystem.remove(filePath) } catch (e) { - console.error(`Error moving file from ${filePath} to ${newFilePath}:`, e); + console.error(`Error moving file from ${filePath} to ${newFilePath}:`, e) } -}; +} io.use(async (socket, next) => { const handshakeSchema = z.object({ @@ -91,17 +96,17 @@ io.use(async (socket, next) => { sandboxId: z.string(), EIO: z.string(), transport: z.string(), - }); + }) - const q = socket.handshake.query; - const parseQuery = handshakeSchema.safeParse(q); + const q = socket.handshake.query + const parseQuery = handshakeSchema.safeParse(q) if (!parseQuery.success) { - next(new Error("Invalid request.")); - return; + next(new Error("Invalid request.")) + return } - const { sandboxId, userId } = parseQuery.data; + const { sandboxId, userId } = parseQuery.data const dbUser = await fetch( `${process.env.DATABASE_WORKER_URL}/api/user?id=${userId}`, { @@ -109,38 +114,41 @@ io.use(async (socket, next) => { Authorization: `${process.env.WORKERS_KEY}`, }, } - ); - const dbUserJSON = (await dbUser.json()) as User; + ) + const dbUserJSON = (await dbUser.json()) as User if (!dbUserJSON) { - next(new Error("DB error.")); - return; + next(new Error("DB error.")) + return } - const sandbox = dbUserJSON.sandbox.find((s) => s.id === sandboxId); + const sandbox = dbUserJSON.sandbox.find((s) => s.id === sandboxId) const sharedSandboxes = dbUserJSON.usersToSandboxes.find( (uts) => uts.sandboxId === sandboxId - ); + ) if (!sandbox && !sharedSandboxes) { - next(new Error("Invalid credentials.")); - return; + next(new Error("Invalid credentials.")) + return } socket.data = { userId, sandboxId: sandboxId, isOwner: sandbox !== undefined, - }; + } - next(); -}); + next() +}) -const lockManager = new LockManager(); +const lockManager = new LockManager() -if (!process.env.DOKKU_HOST) console.error('Environment variable DOKKU_HOST is not defined'); -if (!process.env.DOKKU_USERNAME) console.error('Environment variable DOKKU_USERNAME is not defined'); -if (!process.env.DOKKU_KEY) console.error('Environment variable DOKKU_KEY is not defined'); +if (!process.env.DOKKU_HOST) + console.error("Environment variable DOKKU_HOST is not defined") +if (!process.env.DOKKU_USERNAME) + console.error("Environment variable DOKKU_USERNAME is not defined") +if (!process.env.DOKKU_KEY) + console.error("Environment variable DOKKU_KEY is not defined") const client = process.env.DOKKU_HOST && process.env.DOKKU_KEY && process.env.DOKKU_USERNAME @@ -149,498 +157,576 @@ const client = username: process.env.DOKKU_USERNAME, privateKey: fs.readFileSync(process.env.DOKKU_KEY), }) - : null; -client?.connect(); + : null +client?.connect() -const git = process.env.DOKKU_HOST && process.env.DOKKU_KEY ? new SecureGitClient( - `dokku@${process.env.DOKKU_HOST}`, - process.env.DOKKU_KEY -) : null; +const git = + process.env.DOKKU_HOST && process.env.DOKKU_KEY + ? new SecureGitClient( + `dokku@${process.env.DOKKU_HOST}`, + process.env.DOKKU_KEY + ) + : null io.on("connection", async (socket) => { try { - if (inactivityTimeout) clearTimeout(inactivityTimeout); + if (inactivityTimeout) clearTimeout(inactivityTimeout) const data = socket.data as { - userId: string; - sandboxId: string; - isOwner: boolean; - }; + userId: string + sandboxId: string + isOwner: boolean + } if (data.isOwner) { - isOwnerConnected = true; - connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1; + isOwnerConnected = true + connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1 } else { if (!isOwnerConnected) { - socket.emit("disableAccess", "The sandbox owner is not connected."); - return; + socket.emit("disableAccess", "The sandbox owner is not connected.") + return } } - const createdContainer = await lockManager.acquireLock(data.sandboxId, async () => { - try { - // Start a new container if the container doesn't exist or it timed out. - if (!containers[data.sandboxId] || !(await containers[data.sandboxId].isRunning())) { - containers[data.sandboxId] = await Sandbox.create({ timeoutMs: CONTAINER_TIMEOUT }); - console.log("Created container ", data.sandboxId); - return true; + const createdContainer = await lockManager.acquireLock( + data.sandboxId, + async () => { + try { + // Start a new container if the container doesn't exist or it timed out. + if ( + !containers[data.sandboxId] || + !(await containers[data.sandboxId].isRunning()) + ) { + containers[data.sandboxId] = await Sandbox.create({ + timeoutMs: CONTAINER_TIMEOUT, + }) + console.log("Created container ", data.sandboxId) + return true + } + } catch (e: any) { + console.error(`Error creating container ${data.sandboxId}:`, e) + io.emit("error", `Error: container creation. ${e.message ?? e}`) } - } catch (e: any) { - console.error(`Error creating container ${data.sandboxId}:`, e); - io.emit("error", `Error: container creation. ${e.message ?? e}`); } - }); + ) - const sandboxFiles = await getSandboxFiles(data.sandboxId); - const projectDirectory = path.posix.join(dirName, "projects", data.sandboxId); - const containerFiles = containers[data.sandboxId].files; - const fileWatchers: WatchHandle[] = []; + const sandboxFiles = await getSandboxFiles(data.sandboxId) + const projectDirectory = path.posix.join( + dirName, + "projects", + data.sandboxId + ) + const containerFiles = containers[data.sandboxId].files + const fileWatchers: WatchHandle[] = [] // Change the owner of the project directory to user const fixPermissions = async (projectDirectory: string) => { try { await containers[data.sandboxId].commands.run( `sudo chown -R user "${projectDirectory}"` - ); + ) } catch (e: any) { - console.log("Failed to fix permissions: " + e); + console.log("Failed to fix permissions: " + e) } - }; + } // Check if the given path is a directory const isDirectory = async (projectDirectory: string): Promise => { try { const result = await containers[data.sandboxId].commands.run( `[ -d "${projectDirectory}" ] && echo "true" || echo "false"` - ); - return result.stdout.trim() === "true"; + ) + return result.stdout.trim() === "true" } catch (e: any) { - console.log("Failed to check if directory: " + e); - return false; + console.log("Failed to check if directory: " + e) + return false } - }; + } // Only continue to container setup if a new container was created if (createdContainer) { - // Copy all files from the project to the container const promises = sandboxFiles.fileData.map(async (file) => { try { - const filePath = path.posix.join(dirName, file.id); - const parentDirectory = path.dirname(filePath); + const filePath = path.posix.join(dirName, file.id) + const parentDirectory = path.dirname(filePath) if (!containerFiles.exists(parentDirectory)) { - await containerFiles.makeDir(parentDirectory); + await containerFiles.makeDir(parentDirectory) } - await containerFiles.write(filePath, file.data); + await containerFiles.write(filePath, file.data) } catch (e: any) { - console.log("Failed to create file: " + e); + console.log("Failed to create file: " + e) } - }); - await Promise.all(promises); + }) + await Promise.all(promises) // Make the logged in user the owner of all project files - fixPermissions(projectDirectory); - + fixPermissions(projectDirectory) } // Start filesystem watcher for the project directory - const watchDirectory = async (directory: string): Promise => { + const watchDirectory = async ( + directory: string + ): Promise => { try { - return await containerFiles.watch(directory, async (event: FilesystemEvent) => { - try { - - function removeDirName(path : string, dirName : string) { - return path.startsWith(dirName) ? path.slice(dirName.length) : path; - } - - // This is the absolute file path in the container - const containerFilePath = path.posix.join(directory, event.name); - // This is the file path relative to the home directory - const sandboxFilePath = removeDirName(containerFilePath, dirName + "/"); - // This is the directory being watched relative to the home directory - const sandboxDirectory = removeDirName(directory, dirName + "/"); - - // Helper function to find a folder by id - function findFolderById(files: (TFolder | TFile)[], folderId : string) { - return files.find((file : TFolder | TFile) => file.type === "folder" && file.id === folderId); - } - - // A new file or directory was created. - if (event.type === "create") { - const folder = findFolderById(sandboxFiles.files, sandboxDirectory) as TFolder; - const isDir = await isDirectory(containerFilePath); - - const newItem = isDir - ? { id: sandboxFilePath, name: event.name, type: "folder", children: [] } as TFolder - : { id: sandboxFilePath, name: event.name, type: "file" } as TFile; - - if (folder) { - // If the folder exists, add the new item (file/folder) as a child - folder.children.push(newItem); - } else { - // If folder doesn't exist, add the new item to the root - sandboxFiles.files.push(newItem); + return await containerFiles.watch( + directory, + async (event: FilesystemEvent) => { + try { + function removeDirName(path: string, dirName: string) { + return path.startsWith(dirName) + ? path.slice(dirName.length) + : path } - if (!isDir) { - const fileData = await containers[data.sandboxId].files.read(containerFilePath); - const fileContents = typeof fileData === "string" ? fileData : ""; - sandboxFiles.fileData.push({ id: sandboxFilePath, data: fileContents }); + // This is the absolute file path in the container + const containerFilePath = path.posix.join(directory, event.name) + // This is the file path relative to the home directory + const sandboxFilePath = removeDirName( + containerFilePath, + dirName + "/" + ) + // This is the directory being watched relative to the home directory + const sandboxDirectory = removeDirName(directory, dirName + "/") + + // Helper function to find a folder by id + function findFolderById( + files: (TFolder | TFile)[], + folderId: string + ) { + return files.find( + (file: TFolder | TFile) => + file.type === "folder" && file.id === folderId + ) } - console.log(`Create ${sandboxFilePath}`); - } + // A new file or directory was created. + if (event.type === "create") { + const folder = findFolderById( + sandboxFiles.files, + sandboxDirectory + ) as TFolder + const isDir = await isDirectory(containerFilePath) - // A file or directory was removed or renamed. - else if (event.type === "remove" || event.type == "rename") { - const folder = findFolderById(sandboxFiles.files, sandboxDirectory) as TFolder; - const isDir = await isDirectory(containerFilePath); + const newItem = isDir + ? ({ + id: sandboxFilePath, + name: event.name, + type: "folder", + children: [], + } as TFolder) + : ({ + id: sandboxFilePath, + name: event.name, + type: "file", + } as TFile) - const isFileMatch = (file: TFolder | TFile | TFileData) => file.id === sandboxFilePath || file.id.startsWith(containerFilePath + '/'); + if (folder) { + // If the folder exists, add the new item (file/folder) as a child + folder.children.push(newItem) + } else { + // If folder doesn't exist, add the new item to the root + sandboxFiles.files.push(newItem) + } - if (folder) { - // Remove item from its parent folder - folder.children = folder.children.filter((file: TFolder | TFile) => !isFileMatch(file)); - } else { - // Remove from the root if it's not inside a folder - sandboxFiles.files = sandboxFiles.files.filter((file: TFolder | TFile) => !isFileMatch(file)); + if (!isDir) { + const fileData = await containers[data.sandboxId].files.read( + containerFilePath + ) + const fileContents = + typeof fileData === "string" ? fileData : "" + sandboxFiles.fileData.push({ + id: sandboxFilePath, + data: fileContents, + }) + } + + console.log(`Create ${sandboxFilePath}`) } - // Also remove any corresponding file data - sandboxFiles.fileData = sandboxFiles.fileData.filter((file: TFileData) => !isFileMatch(file)); + // A file or directory was removed or renamed. + else if (event.type === "remove" || event.type == "rename") { + const folder = findFolderById( + sandboxFiles.files, + sandboxDirectory + ) as TFolder + const isDir = await isDirectory(containerFilePath) - console.log(`Removed: ${sandboxFilePath}`); - } + const isFileMatch = (file: TFolder | TFile | TFileData) => + file.id === sandboxFilePath || + file.id.startsWith(containerFilePath + "/") - // The contents of a file were changed. - else if (event.type === "write") { - const folder = findFolderById(sandboxFiles.files, sandboxDirectory) as TFolder; - const fileToWrite = sandboxFiles.fileData.find(file => file.id === sandboxFilePath); + if (folder) { + // Remove item from its parent folder + folder.children = folder.children.filter( + (file: TFolder | TFile) => !isFileMatch(file) + ) + } else { + // Remove from the root if it's not inside a folder + sandboxFiles.files = sandboxFiles.files.filter( + (file: TFolder | TFile) => !isFileMatch(file) + ) + } - if (fileToWrite) { - fileToWrite.data = await containers[data.sandboxId].files.read(containerFilePath); - console.log(`Write to ${sandboxFilePath}`); - } else { - // If the file is part of a folder structure, locate it and update its data - const fileInFolder = folder?.children.find(file => file.id === sandboxFilePath); - if (fileInFolder) { - const fileData = await containers[data.sandboxId].files.read(containerFilePath); - const fileContents = typeof fileData === "string" ? fileData : ""; - sandboxFiles.fileData.push({ id: sandboxFilePath, data: fileContents }); - console.log(`Write to ${sandboxFilePath}`); + // Also remove any corresponding file data + sandboxFiles.fileData = sandboxFiles.fileData.filter( + (file: TFileData) => !isFileMatch(file) + ) + + console.log(`Removed: ${sandboxFilePath}`) + } + + // The contents of a file were changed. + else if (event.type === "write") { + const folder = findFolderById( + sandboxFiles.files, + sandboxDirectory + ) as TFolder + const fileToWrite = sandboxFiles.fileData.find( + (file) => file.id === sandboxFilePath + ) + + if (fileToWrite) { + fileToWrite.data = await containers[ + data.sandboxId + ].files.read(containerFilePath) + console.log(`Write to ${sandboxFilePath}`) + } else { + // If the file is part of a folder structure, locate it and update its data + const fileInFolder = folder?.children.find( + (file) => file.id === sandboxFilePath + ) + if (fileInFolder) { + const fileData = await containers[ + data.sandboxId + ].files.read(containerFilePath) + const fileContents = + typeof fileData === "string" ? fileData : "" + sandboxFiles.fileData.push({ + id: sandboxFilePath, + data: fileContents, + }) + console.log(`Write to ${sandboxFilePath}`) + } } } + + // Tell the client to reload the file list + socket.emit("loaded", sandboxFiles.files) + } catch (error) { + console.error( + `Error handling ${event.type} event for ${event.name}:`, + error + ) } - - // Tell the client to reload the file list - socket.emit("loaded", sandboxFiles.files); - - } catch (error) { - console.error(`Error handling ${event.type} event for ${event.name}:`, error); - } - }, { "timeout": 0 } ) + }, + { timeout: 0 } + ) } catch (error) { - console.error(`Error watching filesystem:`, error); + console.error(`Error watching filesystem:`, error) } - }; + } // Watch the project directory - const handle = await watchDirectory(projectDirectory); + const handle = await watchDirectory(projectDirectory) // Keep track of watch handlers to close later - if (handle) fileWatchers.push(handle); + if (handle) fileWatchers.push(handle) // Watch all subdirectories of the project directory, but not deeper // This also means directories created after the container is created won't be watched - const dirContent = await containerFiles.list(projectDirectory); - await Promise.all(dirContent.map(async (item : EntryInfo) => { - if (item.type === "dir") { - console.log("Watching " + item.path); - // Keep track of watch handlers to close later - const handle = await watchDirectory(item.path); - if (handle) fileWatchers.push(handle); - } - })) - - socket.emit("loaded", sandboxFiles.files); + const dirContent = await containerFiles.list(projectDirectory) + await Promise.all( + dirContent.map(async (item: EntryInfo) => { + if (item.type === "dir") { + console.log("Watching " + item.path) + // Keep track of watch handlers to close later + const handle = await watchDirectory(item.path) + if (handle) fileWatchers.push(handle) + } + }) + ) + + socket.emit("loaded", sandboxFiles.files) socket.on("heartbeat", async () => { try { // This keeps the container alive for another CONTAINER_TIMEOUT seconds. - // The E2B docs are unclear, but the timeout is relative to the time of this method call. - await containers[data.sandboxId].setTimeout(CONTAINER_TIMEOUT); + // The E2B docs are unclear, but the timeout is relative to the time of this method call. + await containers[data.sandboxId].setTimeout(CONTAINER_TIMEOUT) } catch (e: any) { - console.error("Error setting timeout:", e); - io.emit("error", `Error: set timeout. ${e.message ?? e}`); + console.error("Error setting timeout:", e) + io.emit("error", `Error: set timeout. ${e.message ?? e}`) } - }); + }) socket.on("getFile", (fileId: string, callback) => { - console.log(fileId); + console.log(fileId) try { - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; + const file = sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return - callback(file.data); + callback(file.data) } catch (e: any) { - console.error("Error getting file:", e); - io.emit("error", `Error: get file. ${e.message ?? e}`); + console.error("Error getting file:", e) + io.emit("error", `Error: get file. ${e.message ?? e}`) } - }); + }) socket.on("getFolder", async (folderId: string, callback) => { try { - const files = await getFolder(folderId); - callback(files); + const files = await getFolder(folderId) + callback(files) } catch (e: any) { - console.error("Error getting folder:", e); - io.emit("error", `Error: get folder. ${e.message ?? e}`); + console.error("Error getting folder:", e) + io.emit("error", `Error: get folder. ${e.message ?? e}`) } - }); + }) // todo: send diffs + debounce for efficiency socket.on("saveFile", async (fileId: string, body: string) => { - if (!fileId) return; // handles saving when no file is open + if (!fileId) return // handles saving when no file is open try { if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { socket.emit( "error", "Error: file size too large. Please reduce the file size." - ); - return; + ) + return } try { - await saveFileRL.consume(data.userId, 1); - await saveFile(fileId, body); + await saveFileRL.consume(data.userId, 1) + await saveFile(fileId, body) } catch (e) { - io.emit("error", "Rate limited: file saving. Please slow down."); - return; + io.emit("error", "Rate limited: file saving. Please slow down.") + return } - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - file.data = body; + const file = sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return + file.data = body await containers[data.sandboxId].files.write( path.posix.join(dirName, file.id), body - ); - fixPermissions(projectDirectory); + ) + fixPermissions(projectDirectory) } catch (e: any) { - console.error("Error saving file:", e); - io.emit("error", `Error: file saving. ${e.message ?? e}`); + console.error("Error saving file:", e) + io.emit("error", `Error: file saving. ${e.message ?? e}`) } - }); + }) socket.on( "moveFile", async (fileId: string, folderId: string, callback) => { try { - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; + const file = sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return - const parts = fileId.split("/"); - const newFileId = folderId + "/" + parts.pop(); + const parts = fileId.split("/") + const newFileId = folderId + "/" + parts.pop() await moveFile( containers[data.sandboxId].files, path.posix.join(dirName, fileId), path.posix.join(dirName, newFileId) - ); - fixPermissions(projectDirectory); + ) + fixPermissions(projectDirectory) - file.id = newFileId; + file.id = newFileId - await renameFile(fileId, newFileId, file.data); - const newFiles = await getSandboxFiles(data.sandboxId); - callback(newFiles.files); + await renameFile(fileId, newFileId, file.data) + const newFiles = await getSandboxFiles(data.sandboxId) + callback(newFiles.files) } catch (e: any) { - console.error("Error moving file:", e); - io.emit("error", `Error: file moving. ${e.message ?? e}`); + console.error("Error moving file:", e) + io.emit("error", `Error: file moving. ${e.message ?? e}`) } } - ); + ) interface CallbackResponse { - success: boolean; - apps?: string[]; - message?: string; + success: boolean + apps?: string[] + message?: string } socket.on( "list", async (callback: (response: CallbackResponse) => void) => { - console.log("Retrieving apps list..."); + console.log("Retrieving apps list...") try { - if (!client) throw Error("Failed to retrieve apps list: No Dokku client") + if (!client) + throw Error("Failed to retrieve apps list: No Dokku client") callback({ success: true, - apps: await client.listApps() - }); + apps: await client.listApps(), + }) } catch (error) { callback({ success: false, message: "Failed to retrieve apps list", - }); + }) } } - ); + ) socket.on( "deploy", async (callback: (response: CallbackResponse) => void) => { try { // Push the project files to the Dokku server - console.log("Deploying project ${data.sandboxId}..."); + console.log("Deploying project ${data.sandboxId}...") if (!git) throw Error("Failed to retrieve apps list: No git client") // Remove the /project/[id]/ component of each file path: const fixedFilePaths = sandboxFiles.fileData.map((file) => { return { ...file, id: file.id.split("/").slice(2).join("/"), - }; - }); + } + }) // Push all files to Dokku. - await git.pushFiles(fixedFilePaths, data.sandboxId); + await git.pushFiles(fixedFilePaths, data.sandboxId) callback({ success: true, - }); + }) } catch (error) { callback({ success: false, message: "Failed to deploy project: " + error, - }); + }) } } - ); + ) socket.on("createFile", async (name: string, callback) => { try { - const size: number = await getProjectSize(data.sandboxId); + const size: number = await getProjectSize(data.sandboxId) // limit is 200mb if (size > 200 * 1024 * 1024) { io.emit( "error", "Rate limited: project size exceeded. Please delete some files." - ); - callback({ success: false }); - return; + ) + callback({ success: false }) + return } try { - await createFileRL.consume(data.userId, 1); + await createFileRL.consume(data.userId, 1) } catch (e) { - io.emit("error", "Rate limited: file creation. Please slow down."); - return; + io.emit("error", "Rate limited: file creation. Please slow down.") + return } - const id = `projects/${data.sandboxId}/${name}`; + const id = `projects/${data.sandboxId}/${name}` await containers[data.sandboxId].files.write( path.posix.join(dirName, id), "" - ); - fixPermissions(projectDirectory); + ) + fixPermissions(projectDirectory) sandboxFiles.files.push({ id, name, type: "file", - }); + }) sandboxFiles.fileData.push({ id, data: "", - }); + }) - await createFile(id); + await createFile(id) - callback({ success: true }); + callback({ success: true }) } catch (e: any) { - console.error("Error creating file:", e); - io.emit("error", `Error: file creation. ${e.message ?? e}`); + console.error("Error creating file:", e) + io.emit("error", `Error: file creation. ${e.message ?? e}`) } - }); + }) socket.on("createFolder", async (name: string, callback) => { try { try { - await createFolderRL.consume(data.userId, 1); + await createFolderRL.consume(data.userId, 1) } catch (e) { - io.emit("error", "Rate limited: folder creation. Please slow down."); - return; + io.emit("error", "Rate limited: folder creation. Please slow down.") + return } - const id = `projects/${data.sandboxId}/${name}`; + const id = `projects/${data.sandboxId}/${name}` await containers[data.sandboxId].files.makeDir( path.posix.join(dirName, id) - ); + ) - callback(); + callback() } catch (e: any) { - console.error("Error creating folder:", e); - io.emit("error", `Error: folder creation. ${e.message ?? e}`); + console.error("Error creating folder:", e) + io.emit("error", `Error: folder creation. ${e.message ?? e}`) } - }); + }) socket.on("renameFile", async (fileId: string, newName: string) => { try { try { - await renameFileRL.consume(data.userId, 1); + await renameFileRL.consume(data.userId, 1) } catch (e) { - io.emit("error", "Rate limited: file renaming. Please slow down."); - return; + io.emit("error", "Rate limited: file renaming. Please slow down.") + return } - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - file.id = newName; + const file = sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return + file.id = newName - const parts = fileId.split("/"); + const parts = fileId.split("/") const newFileId = - parts.slice(0, parts.length - 1).join("/") + "/" + newName; + parts.slice(0, parts.length - 1).join("/") + "/" + newName await moveFile( containers[data.sandboxId].files, path.posix.join(dirName, fileId), path.posix.join(dirName, newFileId) - ); - fixPermissions(projectDirectory); - await renameFile(fileId, newFileId, file.data); + ) + fixPermissions(projectDirectory) + await renameFile(fileId, newFileId, file.data) } catch (e: any) { - console.error("Error renaming folder:", e); - io.emit("error", `Error: folder renaming. ${e.message ?? e}`); + console.error("Error renaming folder:", e) + io.emit("error", `Error: folder renaming. ${e.message ?? e}`) } - }); + }) socket.on("deleteFile", async (fileId: string, callback) => { try { try { - await deleteFileRL.consume(data.userId, 1); + await deleteFileRL.consume(data.userId, 1) } catch (e) { - io.emit("error", "Rate limited: file deletion. Please slow down."); + io.emit("error", "Rate limited: file deletion. Please slow down.") } - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; + const file = sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return await containers[data.sandboxId].files.remove( path.posix.join(dirName, fileId) - ); + ) sandboxFiles.fileData = sandboxFiles.fileData.filter( (f) => f.id !== fileId - ); + ) - await deleteFile(fileId); + await deleteFile(fileId) - const newFiles = await getSandboxFiles(data.sandboxId); - callback(newFiles.files); + const newFiles = await getSandboxFiles(data.sandboxId) + callback(newFiles.files) } catch (e: any) { - console.error("Error deleting file:", e); - io.emit("error", `Error: file deletion. ${e.message ?? e}`); + console.error("Error deleting file:", e) + io.emit("error", `Error: file deletion. ${e.message ?? e}`) } - }); + }) // todo // socket.on("renameFolder", async (folderId: string, newName: string) => { @@ -648,36 +734,36 @@ io.on("connection", async (socket) => { socket.on("deleteFolder", async (folderId: string, callback) => { try { - const files = await getFolder(folderId); + const files = await getFolder(folderId) await Promise.all( files.map(async (file) => { await containers[data.sandboxId].files.remove( path.posix.join(dirName, file) - ); + ) sandboxFiles.fileData = sandboxFiles.fileData.filter( (f) => f.id !== file - ); + ) - await deleteFile(file); + await deleteFile(file) }) - ); + ) - const newFiles = await getSandboxFiles(data.sandboxId); + const newFiles = await getSandboxFiles(data.sandboxId) - callback(newFiles.files); + callback(newFiles.files) } catch (e: any) { - console.error("Error deleting folder:", e); - io.emit("error", `Error: folder deletion. ${e.message ?? e}`); + console.error("Error deleting folder:", e) + io.emit("error", `Error: folder deletion. ${e.message ?? e}`) } - }); + }) socket.on("createTerminal", async (id: string, callback) => { try { // Note: The number of terminals per window is limited on the frontend, but not backend if (terminals[id]) { - return; + return } await lockManager.acquireLock(data.sandboxId, async () => { @@ -685,95 +771,103 @@ io.on("connection", async (socket) => { terminals[id] = new Terminal(containers[data.sandboxId]) await terminals[id].init({ onData: (responseString: string) => { - io.emit("terminalResponse", { id, data: responseString }); + io.emit("terminalResponse", { id, data: responseString }) function extractPortNumber(inputString: string) { // Remove ANSI escape codes - const cleanedString = inputString.replace(/\x1B\[[0-9;]*m/g, ''); + const cleanedString = inputString.replace( + /\x1B\[[0-9;]*m/g, + "" + ) // Regular expression to match port number - const regex = /http:\/\/localhost:(\d+)/; + const regex = /http:\/\/localhost:(\d+)/ // If a match is found, return the port number - const match = cleanedString.match(regex); - return match ? match[1] : null; + const match = cleanedString.match(regex) + return match ? match[1] : null } - const port = parseInt(extractPortNumber(responseString) ?? ""); + const port = parseInt(extractPortNumber(responseString) ?? "") if (port) { io.emit( "previewURL", "https://" + containers[data.sandboxId].getHost(port) - ); + ) } }, cols: 80, rows: 20, //onExit: () => console.log("Terminal exited", id), - }); + }) - const defaultDirectory = path.posix.join(dirName, "projects", data.sandboxId); + const defaultDirectory = path.posix.join( + dirName, + "projects", + data.sandboxId + ) const defaultCommands = [ `cd "${defaultDirectory}"`, "export PS1='user> '", - "clear" + "clear", ] - for (const command of defaultCommands) await terminals[id].sendData(command + "\r"); + for (const command of defaultCommands) + await terminals[id].sendData(command + "\r") - console.log("Created terminal", id); + console.log("Created terminal", id) } catch (e: any) { - console.error(`Error creating terminal ${id}:`, e); - io.emit("error", `Error: terminal creation. ${e.message ?? e}`); + console.error(`Error creating terminal ${id}:`, e) + io.emit("error", `Error: terminal creation. ${e.message ?? e}`) } - }); + }) - callback(); + callback() } catch (e: any) { - console.error(`Error creating terminal ${id}:`, e); - io.emit("error", `Error: terminal creation. ${e.message ?? e}`); + console.error(`Error creating terminal ${id}:`, e) + io.emit("error", `Error: terminal creation. ${e.message ?? e}`) } - }); + }) socket.on( "resizeTerminal", (dimensions: { cols: number; rows: number }) => { try { Object.values(terminals).forEach((t) => { - t.resize(dimensions); - }); + t.resize(dimensions) + }) } catch (e: any) { - console.error("Error resizing terminal:", e); - io.emit("error", `Error: terminal resizing. ${e.message ?? e}`); + console.error("Error resizing terminal:", e) + io.emit("error", `Error: terminal resizing. ${e.message ?? e}`) } } - ); + ) socket.on("terminalData", async (id: string, data: string) => { try { if (!terminals[id]) { - return; + return } - await terminals[id].sendData(data); + await terminals[id].sendData(data) } catch (e: any) { - console.error("Error writing to terminal:", e); - io.emit("error", `Error: writing to terminal. ${e.message ?? e}`); + console.error("Error writing to terminal:", e) + io.emit("error", `Error: writing to terminal. ${e.message ?? e}`) } - }); + }) socket.on("closeTerminal", async (id: string, callback) => { try { if (!terminals[id]) { - return; + return } - await terminals[id].close(); - delete terminals[id]; + await terminals[id].close() + delete terminals[id] - callback(); + callback() } catch (e: any) { - console.error("Error closing terminal:", e); - io.emit("error", `Error: closing terminal. ${e.message ?? e}`); + console.error("Error closing terminal:", e) + io.emit("error", `Error: closing terminal. ${e.message ?? e}`) } - }); + }) socket.on( "generateCode", @@ -797,50 +891,56 @@ io.on("connection", async (socket) => { userId: data.userId, }), } - ); + ) // Generate code from cloudflare workers AI const generateCodePromise = fetch( - `${process.env.AI_WORKER_URL}/api?fileName=${encodeURIComponent(fileName)}&code=${encodeURIComponent(code)}&line=${encodeURIComponent(line)}&instructions=${encodeURIComponent(instructions)}`, + `${process.env.AI_WORKER_URL}/api?fileName=${encodeURIComponent( + fileName + )}&code=${encodeURIComponent(code)}&line=${encodeURIComponent( + line + )}&instructions=${encodeURIComponent(instructions)}`, { headers: { "Content-Type": "application/json", Authorization: `${process.env.CF_AI_KEY}`, }, } - ); + ) const [fetchResponse, generateCodeResponse] = await Promise.all([ fetchPromise, generateCodePromise, - ]); + ]) - const json = await generateCodeResponse.json(); + const json = await generateCodeResponse.json() - callback({ response: json.response, success: true }); + callback({ response: json.response, success: true }) } catch (e: any) { - console.error("Error generating code:", e); - io.emit("error", `Error: code generation. ${e.message ?? e}`); + console.error("Error generating code:", e) + io.emit("error", `Error: code generation. ${e.message ?? e}`) } } - ); + ) socket.on("disconnect", async () => { try { if (data.isOwner) { - connections[data.sandboxId]--; + connections[data.sandboxId]-- } // Stop watching file changes in the container - Promise.all(fileWatchers.map(async (handle : WatchHandle) => { - await handle.close(); - })); + Promise.all( + fileWatchers.map(async (handle: WatchHandle) => { + await handle.close() + }) + ) if (data.isOwner && connections[data.sandboxId] <= 0) { socket.broadcast.emit( "disableAccess", "The sandbox owner has disconnected." - ); + ) } // const sockets = await io.fetchSockets(); @@ -860,16 +960,16 @@ io.on("connection", async (socket) => { // console.log("number of sockets", sockets.length); // } } catch (e: any) { - console.log("Error disconnecting:", e); - io.emit("error", `Error: disconnecting. ${e.message ?? e}`); + console.log("Error disconnecting:", e) + io.emit("error", `Error: disconnecting. ${e.message ?? e}`) } - }); + }) } catch (e: any) { - console.error("Error connecting:", e); - io.emit("error", `Error: connection. ${e.message ?? e}`); + console.error("Error connecting:", e) + io.emit("error", `Error: connection. ${e.message ?? e}`) } -}); +}) httpServer.listen(port, () => { - console.log(`Server running on port ${port}`); -}); + console.log(`Server running on port ${port}`) +}) diff --git a/backend/server/src/ratelimit.ts b/backend/server/src/ratelimit.ts index f0d99fa..f40ab1e 100644 --- a/backend/server/src/ratelimit.ts +++ b/backend/server/src/ratelimit.ts @@ -30,4 +30,4 @@ export const deleteFileRL = new RateLimiterMemory({ export const deleteFolderRL = new RateLimiterMemory({ points: 1, duration: 2, -}) \ No newline at end of file +}) diff --git a/backend/server/src/types.ts b/backend/server/src/types.ts index b71592a..42ad6d0 100644 --- a/backend/server/src/types.ts +++ b/backend/server/src/types.ts @@ -1,70 +1,70 @@ // DB Types export type User = { - id: string; - name: string; - email: string; - generations: number; - sandbox: Sandbox[]; - usersToSandboxes: UsersToSandboxes[]; -}; + id: string + name: string + email: string + generations: number + sandbox: Sandbox[] + usersToSandboxes: UsersToSandboxes[] +} export type Sandbox = { - id: string; - name: string; - type: "react" | "node"; - visibility: "public" | "private"; - createdAt: Date; - userId: string; - usersToSandboxes: UsersToSandboxes[]; -}; + id: string + name: string + type: "react" | "node" + visibility: "public" | "private" + createdAt: Date + userId: string + usersToSandboxes: UsersToSandboxes[] +} export type UsersToSandboxes = { - userId: string; - sandboxId: string; - sharedOn: Date; -}; + userId: string + sandboxId: string + sharedOn: Date +} export type TFolder = { - id: string; - type: "folder"; - name: string; - children: (TFile | TFolder)[]; -}; + id: string + type: "folder" + name: string + children: (TFile | TFolder)[] +} export type TFile = { - id: string; - type: "file"; - name: string; -}; + id: string + type: "file" + name: string +} export type TFileData = { - id: string; - data: string; -}; + id: string + data: string +} export type R2Files = { - objects: R2FileData[]; - truncated: boolean; - delimitedPrefixes: any[]; -}; + objects: R2FileData[] + truncated: boolean + delimitedPrefixes: any[] +} export type R2FileData = { - storageClass: string; - uploaded: string; - checksums: any; - httpEtag: string; - etag: string; - size: number; - version: string; - key: string; -}; + storageClass: string + uploaded: string + checksums: any + httpEtag: string + etag: string + size: number + version: string + key: string +} export type R2FileBody = R2FileData & { - body: ReadableStream; - bodyUsed: boolean; - arrayBuffer: Promise; - text: Promise; - json: Promise; - blob: Promise; -}; + body: ReadableStream + bodyUsed: boolean + arrayBuffer: Promise + text: Promise + json: Promise + blob: Promise +} diff --git a/backend/server/src/utils.ts b/backend/server/src/utils.ts index 0aebb03..5ae1377 100644 --- a/backend/server/src/utils.ts +++ b/backend/server/src/utils.ts @@ -1,23 +1,23 @@ export class LockManager { - private locks: { [key: string]: Promise }; + private locks: { [key: string]: Promise } constructor() { - this.locks = {}; + this.locks = {} } async acquireLock(key: string, task: () => Promise): Promise { if (!this.locks[key]) { this.locks[key] = new Promise(async (resolve, reject) => { try { - const result = await task(); - resolve(result); + const result = await task() + resolve(result) } catch (error) { - reject(error); + reject(error) } finally { - delete this.locks[key]; + delete this.locks[key] } - }); + }) } - return await this.locks[key]; + return await this.locks[key] } -} \ No newline at end of file +} From cc8e0ce18725dbdc8205a1b89886c7e2b9298f90 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 05:44:30 -0600 Subject: [PATCH 03/10] fix: close all E2B terminals when a sandbox is closed --- backend/server/src/index.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index f659c7f..9f08b3b 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -72,7 +72,6 @@ let isOwnerConnected = false const containers: Record = {} const connections: Record = {} -const terminals: Record = {} const dirName = "/home/user" @@ -210,6 +209,8 @@ io.on("connection", async (socket) => { } ) + const terminals: Record = {} + const sandboxFiles = await getSandboxFiles(data.sandboxId) const projectDirectory = path.posix.join( dirName, @@ -929,6 +930,14 @@ io.on("connection", async (socket) => { connections[data.sandboxId]-- } + // Close all terminals for this connection + await Promise.all( + Object.entries(terminals).map(async ([key, terminal]) => { + await terminal.close() + delete terminals[key] + }) + ) + // Stop watching file changes in the container Promise.all( fileWatchers.map(async (handle: WatchHandle) => { From ce4137d6971d482dcc6594a9d9b9a6d90baa72b1 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 05:45:35 -0600 Subject: [PATCH 04/10] chore: increase timeout for E2B sandboxes --- backend/server/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 9f08b3b..23eab00 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -53,7 +53,7 @@ process.on("unhandledRejection", (reason, promise) => { }) // The amount of time in ms that a container will stay alive without a hearbeat. -const CONTAINER_TIMEOUT = 60_000 +const CONTAINER_TIMEOUT = 120_000 dotenv.config() From 54706314eaa59d8361571682a27a16162fba01de Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 15:12:52 -0600 Subject: [PATCH 05/10] chore: refactor into FileManager and TerminalManager classes --- backend/server/src/FileManager.ts | 423 +++++++++++++++++ backend/server/src/TerminalManager.ts | 81 ++++ backend/server/src/index.ts | 641 ++++---------------------- 3 files changed, 584 insertions(+), 561 deletions(-) create mode 100644 backend/server/src/FileManager.ts create mode 100644 backend/server/src/TerminalManager.ts diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts new file mode 100644 index 0000000..1ef5077 --- /dev/null +++ b/backend/server/src/FileManager.ts @@ -0,0 +1,423 @@ +import { FilesystemEvent, Sandbox, WatchHandle } from "e2b" +import path from "path" +import { + createFile, + deleteFile, + getFolder, + getProjectSize, + getSandboxFiles, + renameFile, + saveFile, +} from "./fileoperations" +import { MAX_BODY_SIZE } from "./ratelimit" +import { TFile, TFileData, TFolder } from "./types" + +export type SandboxFiles = { + files: (TFolder | TFile)[] + fileData: TFileData[] +} + +export class FileManager { + private sandboxId: string + private sandbox: Sandbox + public sandboxFiles: SandboxFiles + private fileWatchers: WatchHandle[] = [] + private dirName = "/home/user" + private refreshFileList: (files: SandboxFiles) => void + + constructor( + sandboxId: string, + sandbox: Sandbox, + refreshFileList: (files: SandboxFiles) => void + ) { + this.sandboxId = sandboxId + this.sandbox = sandbox + this.sandboxFiles = { files: [], fileData: [] } + this.refreshFileList = refreshFileList + } + + async initialize() { + this.sandboxFiles = await getSandboxFiles(this.sandboxId) + const projectDirectory = path.posix.join( + this.dirName, + "projects", + this.sandboxId + ) + // Copy all files from the project to the container + const promises = this.sandboxFiles.fileData.map(async (file) => { + try { + const filePath = path.join(this.dirName, file.id) + const parentDirectory = path.dirname(filePath) + if (!this.sandbox.files.exists(parentDirectory)) { + await this.sandbox.files.makeDir(parentDirectory) + } + await this.sandbox.files.write(filePath, file.data) + } catch (e: any) { + console.log("Failed to create file: " + e) + } + }) + await Promise.all(promises) + + // Make the logged in user the owner of all project files + this.fixPermissions() + + await this.watchDirectory(projectDirectory) + await this.watchSubdirectories(projectDirectory) + } + + // Check if the given path is a directory + private async isDirectory(projectDirectory: string): Promise { + try { + const result = await this.sandbox.commands.run( + `[ -d "${projectDirectory}" ] && echo "true" || echo "false"` + ) + return result.stdout.trim() === "true" + } catch (e: any) { + console.log("Failed to check if directory: " + e) + return false + } + } + + // Change the owner of the project directory to user + private async fixPermissions() { + try { + const projectDirectory = path.posix.join( + this.dirName, + "projects", + this.sandboxId + ) + await this.sandbox.commands.run( + `sudo chown -R user "${projectDirectory}"` + ) + } catch (e: any) { + console.log("Failed to fix permissions: " + e) + } + } + + async watchDirectory(directory: string): Promise { + try { + const handle = await this.sandbox.files.watch( + directory, + async (event: FilesystemEvent) => { + try { + function removeDirName(path: string, dirName: string) { + return path.startsWith(dirName) + ? path.slice(dirName.length) + : path + } + + // This is the absolute file path in the container + const containerFilePath = path.posix.join(directory, event.name) + // This is the file path relative to the home directory + const sandboxFilePath = removeDirName( + containerFilePath, + this.dirName + "/" + ) + // This is the directory being watched relative to the home directory + const sandboxDirectory = removeDirName( + directory, + this.dirName + "/" + ) + + // Helper function to find a folder by id + function findFolderById( + files: (TFolder | TFile)[], + folderId: string + ) { + return files.find( + (file: TFolder | TFile) => + file.type === "folder" && file.id === folderId + ) + } + + // A new file or directory was created. + if (event.type === "create") { + const folder = findFolderById( + this.sandboxFiles.files, + sandboxDirectory + ) as TFolder + const isDir = await this.isDirectory(containerFilePath) + + const newItem = isDir + ? ({ + id: sandboxFilePath, + name: event.name, + type: "folder", + children: [], + } as TFolder) + : ({ + id: sandboxFilePath, + name: event.name, + type: "file", + } as TFile) + + if (folder) { + // If the folder exists, add the new item (file/folder) as a child + folder.children.push(newItem) + } else { + // If folder doesn't exist, add the new item to the root + this.sandboxFiles.files.push(newItem) + } + + if (!isDir) { + const fileData = await this.sandbox.files.read( + containerFilePath + ) + const fileContents = + typeof fileData === "string" ? fileData : "" + this.sandboxFiles.fileData.push({ + id: sandboxFilePath, + data: fileContents, + }) + } + + console.log(`Create ${sandboxFilePath}`) + } + + // A file or directory was removed or renamed. + else if (event.type === "remove" || event.type == "rename") { + const folder = findFolderById( + this.sandboxFiles.files, + sandboxDirectory + ) as TFolder + const isDir = await this.isDirectory(containerFilePath) + + const isFileMatch = (file: TFolder | TFile | TFileData) => + file.id === sandboxFilePath || + file.id.startsWith(containerFilePath + "/") + + if (folder) { + // Remove item from its parent folder + folder.children = folder.children.filter( + (file: TFolder | TFile) => !isFileMatch(file) + ) + } else { + // Remove from the root if it's not inside a folder + this.sandboxFiles.files = this.sandboxFiles.files.filter( + (file: TFolder | TFile) => !isFileMatch(file) + ) + } + + // Also remove any corresponding file data + this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter( + (file: TFileData) => !isFileMatch(file) + ) + + console.log(`Removed: ${sandboxFilePath}`) + } + + // The contents of a file were changed. + else if (event.type === "write") { + const folder = findFolderById( + this.sandboxFiles.files, + sandboxDirectory + ) as TFolder + const fileToWrite = this.sandboxFiles.fileData.find( + (file) => file.id === sandboxFilePath + ) + + if (fileToWrite) { + fileToWrite.data = await this.sandbox.files.read( + containerFilePath + ) + console.log(`Write to ${sandboxFilePath}`) + } else { + // If the file is part of a folder structure, locate it and update its data + const fileInFolder = folder?.children.find( + (file) => file.id === sandboxFilePath + ) + if (fileInFolder) { + const fileData = await this.sandbox.files.read( + containerFilePath + ) + const fileContents = + typeof fileData === "string" ? fileData : "" + this.sandboxFiles.fileData.push({ + id: sandboxFilePath, + data: fileContents, + }) + console.log(`Write to ${sandboxFilePath}`) + } + } + } + + // Tell the client to reload the file list + this.refreshFileList(this.sandboxFiles) + } catch (error) { + console.error( + `Error handling ${event.type} event for ${event.name}:`, + error + ) + } + }, + { timeout: 0 } + ) + this.fileWatchers.push(handle) + return handle + } catch (error) { + console.error(`Error watching filesystem:`, error) + } + } + + async watchSubdirectories(directory: string) { + const dirContent = await this.sandbox.files.list(directory) + await Promise.all( + dirContent.map(async (item) => { + if (item.type === "dir") { + console.log("Watching " + item.path) + await this.watchDirectory(item.path) + } + }) + ) + } + + async getFile(fileId: string): Promise { + const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) + return file?.data + } + + async getFolder(folderId: string): Promise { + return getFolder(folderId) + } + + async saveFile(fileId: string, body: string): Promise { + if (!fileId) return // handles saving when no file is open + + if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { + throw new Error("File size too large. Please reduce the file size.") + } + await saveFile(fileId, body) + const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return + file.data = body + + await this.sandbox.files.write(path.posix.join(this.dirName, file.id), body) + this.fixPermissions() + } + + async moveFile( + fileId: string, + folderId: string + ): Promise<(TFolder | TFile)[]> { + const fileData = this.sandboxFiles.fileData.find((f) => f.id === fileId) + const file = this.sandboxFiles.files.find((f) => f.id === fileId) + if (!fileData || !file) return this.sandboxFiles.files + + const parts = fileId.split("/") + const newFileId = folderId + "/" + parts.pop() + + await this.moveFileInContainer(fileId, newFileId) + + await this.fixPermissions() + + fileData.id = newFileId + file.id = newFileId + + await renameFile(fileId, newFileId, fileData.data) + const newFiles = await getSandboxFiles(this.sandboxId) + return newFiles.files + } + + private async moveFileInContainer(oldPath: string, newPath: string) { + try { + const fileContents = await this.sandbox.files.read( + path.posix.join(this.dirName, oldPath) + ) + await this.sandbox.files.write( + path.posix.join(this.dirName, newPath), + fileContents + ) + await this.sandbox.files.remove(path.posix.join(this.dirName, oldPath)) + } catch (e) { + console.error(`Error moving file from ${oldPath} to ${newPath}:`, e) + } + } + + async createFile(name: string): Promise { + const size: number = await getProjectSize(this.sandboxId) + if (size > 200 * 1024 * 1024) { + throw new Error("Project size exceeded. Please delete some files.") + } + + const id = `projects/${this.sandboxId}/${name}` + + await this.sandbox.files.write(path.posix.join(this.dirName, id), "") + await this.fixPermissions() + + this.sandboxFiles.files.push({ + id, + name, + type: "file", + }) + + this.sandboxFiles.fileData.push({ + id, + data: "", + }) + + await createFile(id) + + return true + } + + async createFolder(name: string): Promise { + const id = `projects/${this.sandboxId}/${name}` + await this.sandbox.files.makeDir(path.posix.join(this.dirName, id)) + } + + async renameFile(fileId: string, newName: string): Promise { + const fileData = this.sandboxFiles.fileData.find((f) => f.id === fileId) + const file = this.sandboxFiles.files.find((f) => f.id === fileId) + if (!fileData || !file) return + + const parts = fileId.split("/") + const newFileId = parts.slice(0, parts.length - 1).join("/") + "/" + newName + + await this.moveFileInContainer(fileId, newFileId) + await this.fixPermissions() + await renameFile(fileId, newFileId, fileData.data) + + fileData.id = newFileId + file.id = newFileId + } + + async deleteFile(fileId: string): Promise<(TFolder | TFile)[]> { + const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) + if (!file) return this.sandboxFiles.files + + await this.sandbox.files.remove(path.posix.join(this.dirName, fileId)) + this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter( + (f) => f.id !== fileId + ) + + await deleteFile(fileId) + + const newFiles = await getSandboxFiles(this.sandboxId) + return newFiles.files + } + + async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> { + const files = await getFolder(folderId) + + await Promise.all( + files.map(async (file) => { + await this.sandbox.files.remove(path.posix.join(this.dirName, file)) + this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter( + (f) => f.id !== file + ) + await deleteFile(file) + }) + ) + + const newFiles = await getSandboxFiles(this.sandboxId) + return newFiles.files + } + + async closeWatchers() { + await Promise.all( + this.fileWatchers.map(async (handle: WatchHandle) => { + await handle.close() + }) + ) + } +} diff --git a/backend/server/src/TerminalManager.ts b/backend/server/src/TerminalManager.ts new file mode 100644 index 0000000..a9bf55c --- /dev/null +++ b/backend/server/src/TerminalManager.ts @@ -0,0 +1,81 @@ +import { Sandbox } from "e2b" +import path from "path" +import { Terminal } from "./Terminal" + +export class TerminalManager { + private sandboxId: string + private sandbox: Sandbox + private terminals: Record = {} + + constructor(sandboxId: string, sandbox: Sandbox) { + this.sandboxId = sandboxId + this.sandbox = sandbox + } + + async createTerminal( + id: string, + onData: (responseString: string) => void + ): Promise { + if (this.terminals[id]) { + return + } + + this.terminals[id] = new Terminal(this.sandbox) + await this.terminals[id].init({ + onData, + cols: 80, + rows: 20, + }) + + const defaultDirectory = path.posix.join( + "/home/user", + "projects", + this.sandboxId + ) + const defaultCommands = [ + `cd "${defaultDirectory}"`, + "export PS1='user> '", + "clear", + ] + for (const command of defaultCommands) { + await this.terminals[id].sendData(command + "\r") + } + + console.log("Created terminal", id) + } + + async resizeTerminal(dimensions: { + cols: number + rows: number + }): Promise { + Object.values(this.terminals).forEach((t) => { + t.resize(dimensions) + }) + } + + async sendTerminalData(id: string, data: string): Promise { + if (!this.terminals[id]) { + return + } + + await this.terminals[id].sendData(data) + } + + async closeTerminal(id: string): Promise { + if (!this.terminals[id]) { + return + } + + await this.terminals[id].close() + delete this.terminals[id] + } + + async closeAllTerminals(): Promise { + await Promise.all( + Object.entries(this.terminals).map(async ([key, terminal]) => { + await terminal.close() + delete this.terminals[key] + }) + ) + } +} diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 23eab00..ffaf067 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -1,44 +1,24 @@ import cors from "cors" import dotenv from "dotenv" +import { Sandbox } from "e2b" import express, { Express } from "express" import fs from "fs" import { createServer } from "http" -import path from "path" import { Server } from "socket.io" -import { DokkuClient } from "./DokkuClient" -import { SecureGitClient } from "./SecureGitClient" - import { z } from "zod" +import { DokkuClient } from "./DokkuClient" +import { FileManager, SandboxFiles } from "./FileManager" import { - createFile, - deleteFile, - getFolder, - getProjectSize, - getSandboxFiles, - renameFile, - saveFile, -} from "./fileoperations" -import { TFile, TFileData, TFolder, User } from "./types" -import { LockManager } from "./utils" - -import { - EntryInfo, - Filesystem, - FilesystemEvent, - Sandbox, - WatchHandle, -} from "e2b" - -import { Terminal } from "./Terminal" - -import { - MAX_BODY_SIZE, createFileRL, createFolderRL, deleteFileRL, renameFileRL, saveFileRL, } from "./ratelimit" +import { SecureGitClient } from "./SecureGitClient" +import { TerminalManager } from "./TerminalManager" +import { User } from "./types" +import { LockManager } from "./utils" process.on("uncaughtException", (error) => { console.error("Uncaught Exception:", error) @@ -67,27 +47,21 @@ const io = new Server(httpServer, { }, }) -let inactivityTimeout: NodeJS.Timeout | null = null -let isOwnerConnected = false +function isOwnerConnected(sandboxId: string): boolean { + return (connections[sandboxId] ?? 0) > 0 +} + +function extractPortNumber(inputString: string): number | null { + const cleanedString = inputString.replace(/\x1B\[[0-9;]*m/g, "") + const regex = /http:\/\/localhost:(\d+)/ + const match = cleanedString.match(regex) + return match ? parseInt(match[1]) : null +} const containers: Record = {} const connections: Record = {} - -const dirName = "/home/user" - -const moveFile = async ( - filesystem: Filesystem, - filePath: string, - newFilePath: string -) => { - try { - const fileContents = await filesystem.read(filePath) - await filesystem.write(newFilePath, fileContents) - await filesystem.remove(filePath) - } catch (e) { - console.error(`Error moving file from ${filePath} to ${newFilePath}:`, e) - } -} +const fileManagers: Record = {} +const terminalManagers: Record = {} io.use(async (socket, next) => { const handshakeSchema = z.object({ @@ -169,8 +143,6 @@ const git = io.on("connection", async (socket) => { try { - if (inactivityTimeout) clearTimeout(inactivityTimeout) - const data = socket.data as { userId: string sandboxId: string @@ -178,10 +150,9 @@ io.on("connection", async (socket) => { } if (data.isOwner) { - isOwnerConnected = true connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1 } else { - if (!isOwnerConnected) { + if (!isOwnerConnected(data.sandboxId)) { socket.emit("disableAccess", "The sandbox owner is not connected.") return } @@ -209,245 +180,28 @@ io.on("connection", async (socket) => { } ) - const terminals: Record = {} - - const sandboxFiles = await getSandboxFiles(data.sandboxId) - const projectDirectory = path.posix.join( - dirName, - "projects", - data.sandboxId - ) - const containerFiles = containers[data.sandboxId].files - const fileWatchers: WatchHandle[] = [] - - // Change the owner of the project directory to user - const fixPermissions = async (projectDirectory: string) => { - try { - await containers[data.sandboxId].commands.run( - `sudo chown -R user "${projectDirectory}"` - ) - } catch (e: any) { - console.log("Failed to fix permissions: " + e) - } + const sendLoadedEvent = (files: SandboxFiles) => { + socket.emit("loaded", files.files) } - // Check if the given path is a directory - const isDirectory = async (projectDirectory: string): Promise => { - try { - const result = await containers[data.sandboxId].commands.run( - `[ -d "${projectDirectory}" ] && echo "true" || echo "false"` - ) - return result.stdout.trim() === "true" - } catch (e: any) { - console.log("Failed to check if directory: " + e) - return false - } - } - - // Only continue to container setup if a new container was created if (createdContainer) { - // Copy all files from the project to the container - const promises = sandboxFiles.fileData.map(async (file) => { - try { - const filePath = path.posix.join(dirName, file.id) - const parentDirectory = path.dirname(filePath) - if (!containerFiles.exists(parentDirectory)) { - await containerFiles.makeDir(parentDirectory) - } - await containerFiles.write(filePath, file.data) - } catch (e: any) { - console.log("Failed to create file: " + e) - } - }) - await Promise.all(promises) - - // Make the logged in user the owner of all project files - fixPermissions(projectDirectory) + fileManagers[data.sandboxId] = new FileManager( + data.sandboxId, + containers[data.sandboxId], + sendLoadedEvent + ) + await fileManagers[data.sandboxId].initialize() + terminalManagers[data.sandboxId] = new TerminalManager( + data.sandboxId, + containers[data.sandboxId] + ) } - // Start filesystem watcher for the project directory - const watchDirectory = async ( - directory: string - ): Promise => { - try { - return await containerFiles.watch( - directory, - async (event: FilesystemEvent) => { - try { - function removeDirName(path: string, dirName: string) { - return path.startsWith(dirName) - ? path.slice(dirName.length) - : path - } + const fileManager = fileManagers[data.sandboxId] + const terminalManager = terminalManagers[data.sandboxId] - // This is the absolute file path in the container - const containerFilePath = path.posix.join(directory, event.name) - // This is the file path relative to the home directory - const sandboxFilePath = removeDirName( - containerFilePath, - dirName + "/" - ) - // This is the directory being watched relative to the home directory - const sandboxDirectory = removeDirName(directory, dirName + "/") - - // Helper function to find a folder by id - function findFolderById( - files: (TFolder | TFile)[], - folderId: string - ) { - return files.find( - (file: TFolder | TFile) => - file.type === "folder" && file.id === folderId - ) - } - - // A new file or directory was created. - if (event.type === "create") { - const folder = findFolderById( - sandboxFiles.files, - sandboxDirectory - ) as TFolder - const isDir = await isDirectory(containerFilePath) - - const newItem = isDir - ? ({ - id: sandboxFilePath, - name: event.name, - type: "folder", - children: [], - } as TFolder) - : ({ - id: sandboxFilePath, - name: event.name, - type: "file", - } as TFile) - - if (folder) { - // If the folder exists, add the new item (file/folder) as a child - folder.children.push(newItem) - } else { - // If folder doesn't exist, add the new item to the root - sandboxFiles.files.push(newItem) - } - - if (!isDir) { - const fileData = await containers[data.sandboxId].files.read( - containerFilePath - ) - const fileContents = - typeof fileData === "string" ? fileData : "" - sandboxFiles.fileData.push({ - id: sandboxFilePath, - data: fileContents, - }) - } - - console.log(`Create ${sandboxFilePath}`) - } - - // A file or directory was removed or renamed. - else if (event.type === "remove" || event.type == "rename") { - const folder = findFolderById( - sandboxFiles.files, - sandboxDirectory - ) as TFolder - const isDir = await isDirectory(containerFilePath) - - const isFileMatch = (file: TFolder | TFile | TFileData) => - file.id === sandboxFilePath || - file.id.startsWith(containerFilePath + "/") - - if (folder) { - // Remove item from its parent folder - folder.children = folder.children.filter( - (file: TFolder | TFile) => !isFileMatch(file) - ) - } else { - // Remove from the root if it's not inside a folder - sandboxFiles.files = sandboxFiles.files.filter( - (file: TFolder | TFile) => !isFileMatch(file) - ) - } - - // Also remove any corresponding file data - sandboxFiles.fileData = sandboxFiles.fileData.filter( - (file: TFileData) => !isFileMatch(file) - ) - - console.log(`Removed: ${sandboxFilePath}`) - } - - // The contents of a file were changed. - else if (event.type === "write") { - const folder = findFolderById( - sandboxFiles.files, - sandboxDirectory - ) as TFolder - const fileToWrite = sandboxFiles.fileData.find( - (file) => file.id === sandboxFilePath - ) - - if (fileToWrite) { - fileToWrite.data = await containers[ - data.sandboxId - ].files.read(containerFilePath) - console.log(`Write to ${sandboxFilePath}`) - } else { - // If the file is part of a folder structure, locate it and update its data - const fileInFolder = folder?.children.find( - (file) => file.id === sandboxFilePath - ) - if (fileInFolder) { - const fileData = await containers[ - data.sandboxId - ].files.read(containerFilePath) - const fileContents = - typeof fileData === "string" ? fileData : "" - sandboxFiles.fileData.push({ - id: sandboxFilePath, - data: fileContents, - }) - console.log(`Write to ${sandboxFilePath}`) - } - } - } - - // Tell the client to reload the file list - socket.emit("loaded", sandboxFiles.files) - } catch (error) { - console.error( - `Error handling ${event.type} event for ${event.name}:`, - error - ) - } - }, - { timeout: 0 } - ) - } catch (error) { - console.error(`Error watching filesystem:`, error) - } - } - - // Watch the project directory - const handle = await watchDirectory(projectDirectory) - // Keep track of watch handlers to close later - if (handle) fileWatchers.push(handle) - - // Watch all subdirectories of the project directory, but not deeper - // This also means directories created after the container is created won't be watched - const dirContent = await containerFiles.list(projectDirectory) - await Promise.all( - dirContent.map(async (item: EntryInfo) => { - if (item.type === "dir") { - console.log("Watching " + item.path) - // Keep track of watch handlers to close later - const handle = await watchDirectory(item.path) - if (handle) fileWatchers.push(handle) - } - }) - ) - - socket.emit("loaded", sandboxFiles.files) + // Load file list from the file manager into the editor + sendLoadedEvent(fileManager.sandboxFiles) socket.on("heartbeat", async () => { try { @@ -460,13 +214,10 @@ io.on("connection", async (socket) => { } }) - socket.on("getFile", (fileId: string, callback) => { - console.log(fileId) + socket.on("getFile", async (fileId: string, callback) => { try { - const file = sandboxFiles.fileData.find((f) => f.id === fileId) - if (!file) return - - callback(file.data) + const fileContent = await fileManager.getFile(fileId) + callback(fileContent) } catch (e: any) { console.error("Error getting file:", e) io.emit("error", `Error: get file. ${e.message ?? e}`) @@ -475,7 +226,7 @@ io.on("connection", async (socket) => { socket.on("getFolder", async (folderId: string, callback) => { try { - const files = await getFolder(folderId) + const files = await fileManager.getFolder(folderId) callback(files) } catch (e: any) { console.error("Error getting folder:", e) @@ -483,35 +234,10 @@ io.on("connection", async (socket) => { } }) - // todo: send diffs + debounce for efficiency socket.on("saveFile", async (fileId: string, body: string) => { - if (!fileId) return // handles saving when no file is open - try { - if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { - socket.emit( - "error", - "Error: file size too large. Please reduce the file size." - ) - return - } - try { - await saveFileRL.consume(data.userId, 1) - await saveFile(fileId, body) - } catch (e) { - io.emit("error", "Rate limited: file saving. Please slow down.") - return - } - - const file = sandboxFiles.fileData.find((f) => f.id === fileId) - if (!file) return - file.data = body - - await containers[data.sandboxId].files.write( - path.posix.join(dirName, file.id), - body - ) - fixPermissions(projectDirectory) + await saveFileRL.consume(data.userId, 1) + await fileManager.saveFile(fileId, body) } catch (e: any) { console.error("Error saving file:", e) io.emit("error", `Error: file saving. ${e.message ?? e}`) @@ -522,24 +248,8 @@ io.on("connection", async (socket) => { "moveFile", async (fileId: string, folderId: string, callback) => { try { - const file = sandboxFiles.fileData.find((f) => f.id === fileId) - if (!file) return - - const parts = fileId.split("/") - const newFileId = folderId + "/" + parts.pop() - - await moveFile( - containers[data.sandboxId].files, - path.posix.join(dirName, fileId), - path.posix.join(dirName, newFileId) - ) - fixPermissions(projectDirectory) - - file.id = newFileId - - await renameFile(fileId, newFileId, file.data) - const newFiles = await getSandboxFiles(data.sandboxId) - callback(newFiles.files) + const newFiles = await fileManager.moveFile(fileId, folderId) + callback(newFiles) } catch (e: any) { console.error("Error moving file:", e) io.emit("error", `Error: file moving. ${e.message ?? e}`) @@ -581,12 +291,14 @@ io.on("connection", async (socket) => { console.log("Deploying project ${data.sandboxId}...") if (!git) throw Error("Failed to retrieve apps list: No git client") // Remove the /project/[id]/ component of each file path: - const fixedFilePaths = sandboxFiles.fileData.map((file) => { - return { - ...file, - id: file.id.split("/").slice(2).join("/"), + const fixedFilePaths = fileManager.sandboxFiles.fileData.map( + (file) => { + return { + ...file, + id: file.id.split("/").slice(2).join("/"), + } } - }) + ) // Push all files to Dokku. await git.pushFiles(fixedFilePaths, data.sandboxId) callback({ @@ -603,46 +315,9 @@ io.on("connection", async (socket) => { socket.on("createFile", async (name: string, callback) => { try { - const size: number = await getProjectSize(data.sandboxId) - // limit is 200mb - if (size > 200 * 1024 * 1024) { - io.emit( - "error", - "Rate limited: project size exceeded. Please delete some files." - ) - callback({ success: false }) - return - } - - try { - await createFileRL.consume(data.userId, 1) - } catch (e) { - io.emit("error", "Rate limited: file creation. Please slow down.") - return - } - - const id = `projects/${data.sandboxId}/${name}` - - await containers[data.sandboxId].files.write( - path.posix.join(dirName, id), - "" - ) - fixPermissions(projectDirectory) - - sandboxFiles.files.push({ - id, - name, - type: "file", - }) - - sandboxFiles.fileData.push({ - id, - data: "", - }) - - await createFile(id) - - callback({ success: true }) + await createFileRL.consume(data.userId, 1) + const success = await fileManager.createFile(name) + callback({ success }) } catch (e: any) { console.error("Error creating file:", e) io.emit("error", `Error: file creation. ${e.message ?? e}`) @@ -651,19 +326,8 @@ io.on("connection", async (socket) => { socket.on("createFolder", async (name: string, callback) => { try { - try { - await createFolderRL.consume(data.userId, 1) - } catch (e) { - io.emit("error", "Rate limited: folder creation. Please slow down.") - return - } - - const id = `projects/${data.sandboxId}/${name}` - - await containers[data.sandboxId].files.makeDir( - path.posix.join(dirName, id) - ) - + await createFolderRL.consume(data.userId, 1) + await fileManager.createFolder(name) callback() } catch (e: any) { console.error("Error creating folder:", e) @@ -673,87 +337,29 @@ io.on("connection", async (socket) => { socket.on("renameFile", async (fileId: string, newName: string) => { try { - try { - await renameFileRL.consume(data.userId, 1) - } catch (e) { - io.emit("error", "Rate limited: file renaming. Please slow down.") - return - } - - const file = sandboxFiles.fileData.find((f) => f.id === fileId) - if (!file) return - file.id = newName - - const parts = fileId.split("/") - const newFileId = - parts.slice(0, parts.length - 1).join("/") + "/" + newName - - await moveFile( - containers[data.sandboxId].files, - path.posix.join(dirName, fileId), - path.posix.join(dirName, newFileId) - ) - fixPermissions(projectDirectory) - await renameFile(fileId, newFileId, file.data) + await renameFileRL.consume(data.userId, 1) + await fileManager.renameFile(fileId, newName) } catch (e: any) { - console.error("Error renaming folder:", e) - io.emit("error", `Error: folder renaming. ${e.message ?? e}`) + console.error("Error renaming file:", e) + io.emit("error", `Error: file renaming. ${e.message ?? e}`) } }) socket.on("deleteFile", async (fileId: string, callback) => { try { - try { - await deleteFileRL.consume(data.userId, 1) - } catch (e) { - io.emit("error", "Rate limited: file deletion. Please slow down.") - } - - const file = sandboxFiles.fileData.find((f) => f.id === fileId) - if (!file) return - - await containers[data.sandboxId].files.remove( - path.posix.join(dirName, fileId) - ) - sandboxFiles.fileData = sandboxFiles.fileData.filter( - (f) => f.id !== fileId - ) - - await deleteFile(fileId) - - const newFiles = await getSandboxFiles(data.sandboxId) - callback(newFiles.files) + await deleteFileRL.consume(data.userId, 1) + const newFiles = await fileManager.deleteFile(fileId) + callback(newFiles) } catch (e: any) { console.error("Error deleting file:", e) io.emit("error", `Error: file deletion. ${e.message ?? e}`) } }) - // todo - // socket.on("renameFolder", async (folderId: string, newName: string) => { - // }); - socket.on("deleteFolder", async (folderId: string, callback) => { try { - const files = await getFolder(folderId) - - await Promise.all( - files.map(async (file) => { - await containers[data.sandboxId].files.remove( - path.posix.join(dirName, file) - ) - - sandboxFiles.fileData = sandboxFiles.fileData.filter( - (f) => f.id !== file - ) - - await deleteFile(file) - }) - ) - - const newFiles = await getSandboxFiles(data.sandboxId) - - callback(newFiles.files) + const newFiles = await fileManager.deleteFolder(folderId) + callback(newFiles) } catch (e: any) { console.error("Error deleting folder:", e) io.emit("error", `Error: folder deletion. ${e.message ?? e}`) @@ -762,64 +368,18 @@ io.on("connection", async (socket) => { socket.on("createTerminal", async (id: string, callback) => { try { - // Note: The number of terminals per window is limited on the frontend, but not backend - if (terminals[id]) { - return - } - await lockManager.acquireLock(data.sandboxId, async () => { - try { - terminals[id] = new Terminal(containers[data.sandboxId]) - await terminals[id].init({ - onData: (responseString: string) => { - io.emit("terminalResponse", { id, data: responseString }) - - function extractPortNumber(inputString: string) { - // Remove ANSI escape codes - const cleanedString = inputString.replace( - /\x1B\[[0-9;]*m/g, - "" - ) - - // Regular expression to match port number - const regex = /http:\/\/localhost:(\d+)/ - // If a match is found, return the port number - const match = cleanedString.match(regex) - return match ? match[1] : null - } - const port = parseInt(extractPortNumber(responseString) ?? "") - if (port) { - io.emit( - "previewURL", - "https://" + containers[data.sandboxId].getHost(port) - ) - } - }, - cols: 80, - rows: 20, - //onExit: () => console.log("Terminal exited", id), - }) - - const defaultDirectory = path.posix.join( - dirName, - "projects", - data.sandboxId - ) - const defaultCommands = [ - `cd "${defaultDirectory}"`, - "export PS1='user> '", - "clear", - ] - for (const command of defaultCommands) - await terminals[id].sendData(command + "\r") - - console.log("Created terminal", id) - } catch (e: any) { - console.error(`Error creating terminal ${id}:`, e) - io.emit("error", `Error: terminal creation. ${e.message ?? e}`) - } + await terminalManager.createTerminal(id, (responseString: string) => { + io.emit("terminalResponse", { id, data: responseString }) + const port = extractPortNumber(responseString) + if (port) { + io.emit( + "previewURL", + "https://" + containers[data.sandboxId].getHost(port) + ) + } + }) }) - callback() } catch (e: any) { console.error(`Error creating terminal ${id}:`, e) @@ -831,9 +391,7 @@ io.on("connection", async (socket) => { "resizeTerminal", (dimensions: { cols: number; rows: number }) => { try { - Object.values(terminals).forEach((t) => { - t.resize(dimensions) - }) + terminalManager.resizeTerminal(dimensions) } catch (e: any) { console.error("Error resizing terminal:", e) io.emit("error", `Error: terminal resizing. ${e.message ?? e}`) @@ -843,11 +401,7 @@ io.on("connection", async (socket) => { socket.on("terminalData", async (id: string, data: string) => { try { - if (!terminals[id]) { - return - } - - await terminals[id].sendData(data) + await terminalManager.sendTerminalData(id, data) } catch (e: any) { console.error("Error writing to terminal:", e) io.emit("error", `Error: writing to terminal. ${e.message ?? e}`) @@ -856,13 +410,7 @@ io.on("connection", async (socket) => { socket.on("closeTerminal", async (id: string, callback) => { try { - if (!terminals[id]) { - return - } - - await terminals[id].close() - delete terminals[id] - + await terminalManager.closeTerminal(id) callback() } catch (e: any) { console.error("Error closing terminal:", e) @@ -930,20 +478,8 @@ io.on("connection", async (socket) => { connections[data.sandboxId]-- } - // Close all terminals for this connection - await Promise.all( - Object.entries(terminals).map(async ([key, terminal]) => { - await terminal.close() - delete terminals[key] - }) - ) - - // Stop watching file changes in the container - Promise.all( - fileWatchers.map(async (handle: WatchHandle) => { - await handle.close() - }) - ) + await terminalManager.closeAllTerminals() + await fileManager.closeWatchers() if (data.isOwner && connections[data.sandboxId] <= 0) { socket.broadcast.emit( @@ -951,23 +487,6 @@ io.on("connection", async (socket) => { "The sandbox owner has disconnected." ) } - - // const sockets = await io.fetchSockets(); - // if (inactivityTimeout) { - // clearTimeout(inactivityTimeout); - // } - // if (sockets.length === 0) { - // console.log("STARTING TIMER"); - // inactivityTimeout = setTimeout(() => { - // io.fetchSockets().then(async (sockets) => { - // if (sockets.length === 0) { - // console.log("Server stopped", res); - // } - // }); - // }, 20000); - // } else { - // console.log("number of sockets", sockets.length); - // } } catch (e: any) { console.log("Error disconnecting:", e) io.emit("error", `Error: disconnecting. ${e.message ?? e}`) From 7722c533a4e17a02afdf042ed4756a5b2e8c9fb5 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 15:16:24 -0600 Subject: [PATCH 06/10] chore: add comments --- backend/server/src/DokkuClient.ts | 9 +++++- backend/server/src/FileManager.ts | 43 +++++++++++++++++-------- backend/server/src/SSHSocketClient.ts | 10 +++++- backend/server/src/index.ts | 46 ++++++++++++++++++++++++++- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/backend/server/src/DokkuClient.ts b/backend/server/src/DokkuClient.ts index e2e9911..38c559d 100644 --- a/backend/server/src/DokkuClient.ts +++ b/backend/server/src/DokkuClient.ts @@ -1,15 +1,19 @@ import { SSHConfig, SSHSocketClient } from "./SSHSocketClient" +// Interface for the response structure from Dokku commands export interface DokkuResponse { ok: boolean output: string } +// DokkuClient class extends SSHSocketClient to interact with Dokku via SSH export class DokkuClient extends SSHSocketClient { constructor(config: SSHConfig) { + // Initialize with Dokku daemon socket path super(config, "/var/run/dokku-daemon/dokku-daemon.sock") } + // Send a command to Dokku and parse the response async sendCommand(command: string): Promise { try { const response = await this.sendData(command) @@ -18,15 +22,18 @@ export class DokkuClient extends SSHSocketClient { throw new Error("Received data is not a string") } + // Parse the JSON response from Dokku return JSON.parse(response) } catch (error: any) { throw new Error(`Failed to send command: ${error.message}`) } } + // List all deployed Dokku apps async listApps(): Promise { const response = await this.sendCommand("apps:list") - return response.output.split("\n").slice(1) // Split by newline and ignore the first line (header) + // Split the output by newline and remove the header + return response.output.split("\n").slice(1) } } diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index 1ef5077..2ac1a80 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -12,11 +12,13 @@ import { import { MAX_BODY_SIZE } from "./ratelimit" import { TFile, TFileData, TFolder } from "./types" +// Define the structure for sandbox files export type SandboxFiles = { files: (TFolder | TFile)[] fileData: TFileData[] } +// FileManager class to handle file operations in a sandbox export class FileManager { private sandboxId: string private sandbox: Sandbox @@ -25,6 +27,7 @@ export class FileManager { private dirName = "/home/user" private refreshFileList: (files: SandboxFiles) => void + // Constructor to initialize the FileManager constructor( sandboxId: string, sandbox: Sandbox, @@ -36,6 +39,7 @@ export class FileManager { this.refreshFileList = refreshFileList } + // Initialize the FileManager async initialize() { this.sandboxFiles = await getSandboxFiles(this.sandboxId) const projectDirectory = path.posix.join( @@ -94,6 +98,7 @@ export class FileManager { } } + // Watch a directory for changes async watchDirectory(directory: string): Promise { try { const handle = await this.sandbox.files.watch( @@ -130,7 +135,7 @@ export class FileManager { ) } - // A new file or directory was created. + // Handle file/directory creation event if (event.type === "create") { const folder = findFolderById( this.sandboxFiles.files, @@ -140,16 +145,16 @@ export class FileManager { const newItem = isDir ? ({ - id: sandboxFilePath, - name: event.name, - type: "folder", - children: [], - } as TFolder) + id: sandboxFilePath, + name: event.name, + type: "folder", + children: [], + } as TFolder) : ({ - id: sandboxFilePath, - name: event.name, - type: "file", - } as TFile) + id: sandboxFilePath, + name: event.name, + type: "file", + } as TFile) if (folder) { // If the folder exists, add the new item (file/folder) as a child @@ -174,7 +179,7 @@ export class FileManager { console.log(`Create ${sandboxFilePath}`) } - // A file or directory was removed or renamed. + // Handle file/directory removal or rename event else if (event.type === "remove" || event.type == "rename") { const folder = findFolderById( this.sandboxFiles.files, @@ -206,7 +211,7 @@ export class FileManager { console.log(`Removed: ${sandboxFilePath}`) } - // The contents of a file were changed. + // Handle file write event else if (event.type === "write") { const folder = findFolderById( this.sandboxFiles.files, @@ -259,6 +264,7 @@ export class FileManager { } } + // Watch subdirectories recursively async watchSubdirectories(directory: string) { const dirContent = await this.sandbox.files.list(directory) await Promise.all( @@ -271,15 +277,18 @@ export class FileManager { ) } + // Get file content async getFile(fileId: string): Promise { const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) return file?.data } + // Get folder content async getFolder(folderId: string): Promise { return getFolder(folderId) } + // Save file content async saveFile(fileId: string, body: string): Promise { if (!fileId) return // handles saving when no file is open @@ -295,6 +304,7 @@ export class FileManager { this.fixPermissions() } + // Move a file to a different folder async moveFile( fileId: string, folderId: string @@ -318,6 +328,7 @@ export class FileManager { return newFiles.files } + // Move a file within the container private async moveFileInContainer(oldPath: string, newPath: string) { try { const fileContents = await this.sandbox.files.read( @@ -333,6 +344,7 @@ export class FileManager { } } + // Create a new file async createFile(name: string): Promise { const size: number = await getProjectSize(this.sandboxId) if (size > 200 * 1024 * 1024) { @@ -360,11 +372,13 @@ export class FileManager { return true } + // Create a new folder async createFolder(name: string): Promise { const id = `projects/${this.sandboxId}/${name}` await this.sandbox.files.makeDir(path.posix.join(this.dirName, id)) } + // Rename a file async renameFile(fileId: string, newName: string): Promise { const fileData = this.sandboxFiles.fileData.find((f) => f.id === fileId) const file = this.sandboxFiles.files.find((f) => f.id === fileId) @@ -381,6 +395,7 @@ export class FileManager { file.id = newFileId } + // Delete a file async deleteFile(fileId: string): Promise<(TFolder | TFile)[]> { const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) if (!file) return this.sandboxFiles.files @@ -396,6 +411,7 @@ export class FileManager { return newFiles.files } + // Delete a folder async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> { const files = await getFolder(folderId) @@ -413,6 +429,7 @@ export class FileManager { return newFiles.files } + // Close all file watchers async closeWatchers() { await Promise.all( this.fileWatchers.map(async (handle: WatchHandle) => { @@ -420,4 +437,4 @@ export class FileManager { }) ) } -} +} \ No newline at end of file diff --git a/backend/server/src/SSHSocketClient.ts b/backend/server/src/SSHSocketClient.ts index c653b23..0fe4152 100644 --- a/backend/server/src/SSHSocketClient.ts +++ b/backend/server/src/SSHSocketClient.ts @@ -1,5 +1,6 @@ import { Client } from "ssh2" +// Interface defining the configuration for SSH connection export interface SSHConfig { host: string port?: number @@ -7,25 +8,29 @@ export interface SSHConfig { privateKey: Buffer } +// Class to handle SSH connections and communicate with a Unix socket export class SSHSocketClient { private conn: Client private config: SSHConfig private socketPath: string private isConnected: boolean = false + // Constructor initializes the SSH client and sets up configuration constructor(config: SSHConfig, socketPath: string) { this.conn = new Client() - this.config = { ...config, port: 22 } + this.config = { ...config, port: 22 } // Default port to 22 if not provided this.socketPath = socketPath this.setupTerminationHandlers() } + // Set up handlers for graceful termination private setupTerminationHandlers() { process.on("SIGINT", this.closeConnection.bind(this)) process.on("SIGTERM", this.closeConnection.bind(this)) } + // Method to close the SSH connection private closeConnection() { console.log("Closing SSH connection...") this.conn.end() @@ -33,6 +38,7 @@ export class SSHSocketClient { process.exit(0) } + // Method to establish the SSH connection connect(): Promise { return new Promise((resolve, reject) => { this.conn @@ -54,6 +60,7 @@ export class SSHSocketClient { }) } + // Method to send data through the SSH connection to the Unix socket sendData(data: string): Promise { return new Promise((resolve, reject) => { if (!this.isConnected) { @@ -61,6 +68,7 @@ export class SSHSocketClient { return } + // Use netcat to send data to the Unix socket this.conn.exec( `echo "${data}" | nc -U ${this.socketPath}`, (err, stream) => { diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index ffaf067..9d82c4e 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -20,12 +20,14 @@ import { TerminalManager } from "./TerminalManager" import { User } from "./types" import { LockManager } from "./utils" +// Handle uncaught exceptions process.on("uncaughtException", (error) => { console.error("Uncaught Exception:", error) // Do not exit the process // You can add additional logging or recovery logic here }) +// Handle unhandled promise rejections process.on("unhandledRejection", (reason, promise) => { console.error("Unhandled Rejection at:", promise, "reason:", reason) // Do not exit the process @@ -35,8 +37,10 @@ process.on("unhandledRejection", (reason, promise) => { // The amount of time in ms that a container will stay alive without a hearbeat. const CONTAINER_TIMEOUT = 120_000 +// Load environment variables dotenv.config() +// Initialize Express app and create HTTP server const app: Express = express() const port = process.env.PORT || 4000 app.use(cors()) @@ -47,10 +51,12 @@ const io = new Server(httpServer, { }, }) +// Check if the sandbox owner is connected function isOwnerConnected(sandboxId: string): boolean { return (connections[sandboxId] ?? 0) > 0 } +// Extract port number from a string function extractPortNumber(inputString: string): number | null { const cleanedString = inputString.replace(/\x1B\[[0-9;]*m/g, "") const regex = /http:\/\/localhost:(\d+)/ @@ -58,12 +64,15 @@ function extractPortNumber(inputString: string): number | null { return match ? parseInt(match[1]) : null } +// Initialize containers and managers const containers: Record = {} const connections: Record = {} const fileManagers: Record = {} const terminalManagers: Record = {} +// Middleware for socket authentication io.use(async (socket, next) => { + // Define the schema for handshake query validation const handshakeSchema = z.object({ userId: z.string(), sandboxId: z.string(), @@ -74,12 +83,14 @@ io.use(async (socket, next) => { const q = socket.handshake.query const parseQuery = handshakeSchema.safeParse(q) + // Check if the query is valid according to the schema if (!parseQuery.success) { next(new Error("Invalid request.")) return } const { sandboxId, userId } = parseQuery.data + // Fetch user data from the database const dbUser = await fetch( `${process.env.DATABASE_WORKER_URL}/api/user?id=${userId}`, { @@ -90,32 +101,39 @@ io.use(async (socket, next) => { ) const dbUserJSON = (await dbUser.json()) as User + // Check if user data was retrieved successfully if (!dbUserJSON) { next(new Error("DB error.")) return } + // Check if the user owns the sandbox or has shared access const sandbox = dbUserJSON.sandbox.find((s) => s.id === sandboxId) const sharedSandboxes = dbUserJSON.usersToSandboxes.find( (uts) => uts.sandboxId === sandboxId ) + // If user doesn't own or have shared access to the sandbox, deny access if (!sandbox && !sharedSandboxes) { next(new Error("Invalid credentials.")) return } + // Set socket data with user information socket.data = { userId, sandboxId: sandboxId, isOwner: sandbox !== undefined, } + // Allow the connection next() }) +// Initialize lock manager const lockManager = new LockManager() +// Check for required environment variables if (!process.env.DOKKU_HOST) console.error("Environment variable DOKKU_HOST is not defined") if (!process.env.DOKKU_USERNAME) @@ -123,6 +141,7 @@ if (!process.env.DOKKU_USERNAME) if (!process.env.DOKKU_KEY) console.error("Environment variable DOKKU_KEY is not defined") +// Initialize Dokku client const client = process.env.DOKKU_HOST && process.env.DOKKU_KEY && process.env.DOKKU_USERNAME ? new DokkuClient({ @@ -133,6 +152,7 @@ const client = : null client?.connect() +// Initialize Git client used to deploy Dokku apps const git = process.env.DOKKU_HOST && process.env.DOKKU_KEY ? new SecureGitClient( @@ -141,6 +161,7 @@ const git = ) : null +// Handle socket connections io.on("connection", async (socket) => { try { const data = socket.data as { @@ -149,6 +170,7 @@ io.on("connection", async (socket) => { isOwner: boolean } + // Handle connection based on user type (owner or not) if (data.isOwner) { connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1 } else { @@ -158,6 +180,7 @@ io.on("connection", async (socket) => { } } + // Create or retrieve container const createdContainer = await lockManager.acquireLock( data.sandboxId, async () => { @@ -180,10 +203,12 @@ io.on("connection", async (socket) => { } ) + // Function to send loaded event const sendLoadedEvent = (files: SandboxFiles) => { socket.emit("loaded", files.files) } + // Initialize file and terminal managers if container was created if (createdContainer) { fileManagers[data.sandboxId] = new FileManager( data.sandboxId, @@ -203,6 +228,7 @@ io.on("connection", async (socket) => { // Load file list from the file manager into the editor sendLoadedEvent(fileManager.sandboxFiles) + // Handle various socket events (heartbeat, file operations, terminal operations, etc.) socket.on("heartbeat", async () => { try { // This keeps the container alive for another CONTAINER_TIMEOUT seconds. @@ -214,6 +240,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to get file content socket.on("getFile", async (fileId: string, callback) => { try { const fileContent = await fileManager.getFile(fileId) @@ -224,6 +251,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to get folder contents socket.on("getFolder", async (folderId: string, callback) => { try { const files = await fileManager.getFolder(folderId) @@ -234,6 +262,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to save file socket.on("saveFile", async (fileId: string, body: string) => { try { await saveFileRL.consume(data.userId, 1) @@ -244,6 +273,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to move file socket.on( "moveFile", async (fileId: string, folderId: string, callback) => { @@ -263,6 +293,7 @@ io.on("connection", async (socket) => { message?: string } + // Handle request to list apps socket.on( "list", async (callback: (response: CallbackResponse) => void) => { @@ -283,6 +314,7 @@ io.on("connection", async (socket) => { } ) + // Handle request to deploy project socket.on( "deploy", async (callback: (response: CallbackResponse) => void) => { @@ -313,6 +345,7 @@ io.on("connection", async (socket) => { } ) + // Handle request to create a new file socket.on("createFile", async (name: string, callback) => { try { await createFileRL.consume(data.userId, 1) @@ -324,6 +357,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to create a new folder socket.on("createFolder", async (name: string, callback) => { try { await createFolderRL.consume(data.userId, 1) @@ -335,6 +369,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to rename a file socket.on("renameFile", async (fileId: string, newName: string) => { try { await renameFileRL.consume(data.userId, 1) @@ -345,6 +380,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to delete a file socket.on("deleteFile", async (fileId: string, callback) => { try { await deleteFileRL.consume(data.userId, 1) @@ -356,6 +392,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to delete a folder socket.on("deleteFolder", async (folderId: string, callback) => { try { const newFiles = await fileManager.deleteFolder(folderId) @@ -366,6 +403,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to create a new terminal socket.on("createTerminal", async (id: string, callback) => { try { await lockManager.acquireLock(data.sandboxId, async () => { @@ -387,6 +425,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to resize terminal socket.on( "resizeTerminal", (dimensions: { cols: number; rows: number }) => { @@ -399,6 +438,7 @@ io.on("connection", async (socket) => { } ) + // Handle terminal input data socket.on("terminalData", async (id: string, data: string) => { try { await terminalManager.sendTerminalData(id, data) @@ -408,6 +448,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to close terminal socket.on("closeTerminal", async (id: string, callback) => { try { await terminalManager.closeTerminal(id) @@ -418,6 +459,7 @@ io.on("connection", async (socket) => { } }) + // Handle request to generate code socket.on( "generateCode", async ( @@ -442,7 +484,7 @@ io.on("connection", async (socket) => { } ) - // Generate code from cloudflare workers AI + // Generate code from Cloudflare Workers AI const generateCodePromise = fetch( `${process.env.AI_WORKER_URL}/api?fileName=${encodeURIComponent( fileName @@ -472,6 +514,7 @@ io.on("connection", async (socket) => { } ) + // Handle socket disconnection socket.on("disconnect", async () => { try { if (data.isOwner) { @@ -498,6 +541,7 @@ io.on("connection", async (socket) => { } }) +// Start the server httpServer.listen(port, () => { console.log(`Server running on port ${port}`) }) From fe0adb8e84b27abbee3a7e05a311353c328e2799 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 15:43:18 -0600 Subject: [PATCH 07/10] chore: refactor into AIWorker class --- backend/server/src/AIWorker.ts | 77 ++++++++++++++++++++++++++++++++++ backend/server/src/index.ts | 52 +++++++---------------- 2 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 backend/server/src/AIWorker.ts diff --git a/backend/server/src/AIWorker.ts b/backend/server/src/AIWorker.ts new file mode 100644 index 0000000..aed1aab --- /dev/null +++ b/backend/server/src/AIWorker.ts @@ -0,0 +1,77 @@ +// AIWorker class for handling AI-related operations +export class AIWorker { + private aiWorkerUrl: string + private cfAiKey: string + private databaseWorkerUrl: string + private workersKey: string + + // Constructor to initialize AIWorker with necessary URLs and keys + constructor( + aiWorkerUrl: string, + cfAiKey: string, + databaseWorkerUrl: string, + workersKey: string + ) { + this.aiWorkerUrl = aiWorkerUrl + this.cfAiKey = cfAiKey + this.databaseWorkerUrl = databaseWorkerUrl + this.workersKey = workersKey + } + + // Method to generate code based on user input + async generateCode( + userId: string, + fileName: string, + code: string, + line: number, + instructions: string + ): Promise<{ response: string; success: boolean }> { + try { + // Fetch request to the database worker + const fetchPromise = fetch( + `${this.databaseWorkerUrl}/api/sandbox/generate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${this.workersKey}`, + }, + body: JSON.stringify({ + userId: userId, + }), + } + ) + + // Fetch request to the AI worker for code generation + const generateCodePromise = fetch( + `${this.aiWorkerUrl}/api?fileName=${encodeURIComponent( + fileName + )}&code=${encodeURIComponent(code)}&line=${encodeURIComponent( + line + )}&instructions=${encodeURIComponent(instructions)}`, + { + headers: { + "Content-Type": "application/json", + Authorization: `${this.cfAiKey}`, + }, + } + ) + + // Wait for both fetch requests to complete + const [fetchResponse, generateCodeResponse] = await Promise.all([ + fetchPromise, + generateCodePromise, + ]) + + // Parse the response from the AI worker + const json = await generateCodeResponse.json() + + // Return the generated code response + return { response: json.response, success: true } + } catch (e: any) { + // Log and throw an error if code generation fails + console.error("Error generating code:", e) + throw new Error(`Error: code generation. ${e.message ?? e}`) + } + } +} diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 9d82c4e..2d9d42b 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -6,6 +6,7 @@ import fs from "fs" import { createServer } from "http" import { Server } from "socket.io" import { z } from "zod" +import { AIWorker } from "./AIWorker" import { DokkuClient } from "./DokkuClient" import { FileManager, SandboxFiles } from "./FileManager" import { @@ -161,6 +162,14 @@ const git = ) : null +// Add this near the top of the file, after other initializations +const aiWorker = new AIWorker( + process.env.AI_WORKER_URL!, + process.env.CF_AI_KEY!, + process.env.DATABASE_WORKER_URL!, + process.env.WORKERS_KEY! +) + // Handle socket connections io.on("connection", async (socket) => { try { @@ -470,43 +479,14 @@ io.on("connection", async (socket) => { callback ) => { try { - const fetchPromise = fetch( - `${process.env.DATABASE_WORKER_URL}/api/sandbox/generate`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ - userId: data.userId, - }), - } + const result = await aiWorker.generateCode( + data.userId, + fileName, + code, + line, + instructions ) - - // Generate code from Cloudflare Workers AI - const generateCodePromise = fetch( - `${process.env.AI_WORKER_URL}/api?fileName=${encodeURIComponent( - fileName - )}&code=${encodeURIComponent(code)}&line=${encodeURIComponent( - line - )}&instructions=${encodeURIComponent(instructions)}`, - { - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.CF_AI_KEY}`, - }, - } - ) - - const [fetchResponse, generateCodeResponse] = await Promise.all([ - fetchPromise, - generateCodePromise, - ]) - - const json = await generateCodeResponse.json() - - callback({ response: json.response, success: true }) + callback(result) } catch (e: any) { console.error("Error generating code:", e) io.emit("error", `Error: code generation. ${e.message ?? e}`) From ae38a77759d6c9668c9a6b88c43e54af45b8ed6c Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 16:23:31 -0600 Subject: [PATCH 08/10] chore: refactor into RemoteFileStorage --- backend/server/src/FileManager.ts | 89 ++++++++++--- backend/server/src/RemoteFileStorage.ts | 117 ++++++++++++++++ backend/server/src/fileoperations.ts | 170 ------------------------ 3 files changed, 187 insertions(+), 189 deletions(-) create mode 100644 backend/server/src/RemoteFileStorage.ts delete mode 100644 backend/server/src/fileoperations.ts diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index 2ac1a80..d615c43 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -1,14 +1,6 @@ import { FilesystemEvent, Sandbox, WatchHandle } from "e2b" import path from "path" -import { - createFile, - deleteFile, - getFolder, - getProjectSize, - getSandboxFiles, - renameFile, - saveFile, -} from "./fileoperations" +import RemoteFileStorage from "./RemoteFileStorage" import { MAX_BODY_SIZE } from "./ratelimit" import { TFile, TFileData, TFolder } from "./types" @@ -18,6 +10,65 @@ export type SandboxFiles = { fileData: TFileData[] } +const processFiles = async (paths: string[], id: string) => { + const root: TFolder = { id: "/", type: "folder", name: "/", children: [] } + const fileData: TFileData[] = [] + + paths.forEach((path) => { + const allParts = path.split("/") + if (allParts[1] !== id) { + return + } + + const parts = allParts.slice(2) + let current: TFolder = root + + for (let i = 0; i < parts.length; i++) { + const part = parts[i] + const isFile = i === parts.length - 1 && part.length + const existing = current.children.find((child) => child.name === part) + + if (existing) { + if (!isFile) { + current = existing as TFolder + } + } else { + if (isFile) { + const file: TFile = { id: path, type: "file", name: part } + current.children.push(file) + fileData.push({ id: path, data: "" }) + } else { + const folder: TFolder = { + // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css + id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, + type: "folder", + name: part, + children: [], + } + current.children.push(folder) + current = folder + } + } + } + }) + + await Promise.all( + fileData.map(async (file) => { + const data = await RemoteFileStorage.fetchFileContent(file.id) + file.data = data + }) + ) + + return { + files: root.children, + fileData, + } +} + +const getSandboxFiles = async (id: string) => { + return await processFiles(await RemoteFileStorage.getSandboxPaths(id), id) +} + // FileManager class to handle file operations in a sandbox export class FileManager { private sandboxId: string @@ -285,7 +336,7 @@ export class FileManager { // Get folder content async getFolder(folderId: string): Promise { - return getFolder(folderId) + return RemoteFileStorage.getFolder(folderId) } // Save file content @@ -295,7 +346,7 @@ export class FileManager { if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { throw new Error("File size too large. Please reduce the file size.") } - await saveFile(fileId, body) + await RemoteFileStorage.saveFile(fileId, body) const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) if (!file) return file.data = body @@ -323,7 +374,7 @@ export class FileManager { fileData.id = newFileId file.id = newFileId - await renameFile(fileId, newFileId, fileData.data) + await RemoteFileStorage.renameFile(fileId, newFileId, fileData.data) const newFiles = await getSandboxFiles(this.sandboxId) return newFiles.files } @@ -346,7 +397,7 @@ export class FileManager { // Create a new file async createFile(name: string): Promise { - const size: number = await getProjectSize(this.sandboxId) + const size: number = await RemoteFileStorage.getProjectSize(this.sandboxId) if (size > 200 * 1024 * 1024) { throw new Error("Project size exceeded. Please delete some files.") } @@ -367,7 +418,7 @@ export class FileManager { data: "", }) - await createFile(id) + await RemoteFileStorage.createFile(id) return true } @@ -389,7 +440,7 @@ export class FileManager { await this.moveFileInContainer(fileId, newFileId) await this.fixPermissions() - await renameFile(fileId, newFileId, fileData.data) + await RemoteFileStorage.renameFile(fileId, newFileId, fileData.data) fileData.id = newFileId file.id = newFileId @@ -405,7 +456,7 @@ export class FileManager { (f) => f.id !== fileId ) - await deleteFile(fileId) + await RemoteFileStorage.deleteFile(fileId) const newFiles = await getSandboxFiles(this.sandboxId) return newFiles.files @@ -413,7 +464,7 @@ export class FileManager { // Delete a folder async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> { - const files = await getFolder(folderId) + const files = await RemoteFileStorage.getFolder(folderId) await Promise.all( files.map(async (file) => { @@ -421,7 +472,7 @@ export class FileManager { this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter( (f) => f.id !== file ) - await deleteFile(file) + await RemoteFileStorage.deleteFile(file) }) ) @@ -437,4 +488,4 @@ export class FileManager { }) ) } -} \ No newline at end of file +} diff --git a/backend/server/src/RemoteFileStorage.ts b/backend/server/src/RemoteFileStorage.ts new file mode 100644 index 0000000..e5ed4b2 --- /dev/null +++ b/backend/server/src/RemoteFileStorage.ts @@ -0,0 +1,117 @@ +import * as dotenv from "dotenv" +import { R2Files } from "./types" + +dotenv.config() + +export const RemoteFileStorage = { + getSandboxPaths: async (id: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + const data: R2Files = await res.json() + + return data.objects.map((obj) => obj.key) + }, + + getFolder: async (folderId: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + const data: R2Files = await res.json() + + return data.objects.map((obj) => obj.key) + }, + + fetchFileContent: async (fileId: string): Promise => { + try { + const fileRes = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + return await fileRes.text() + } catch (error) { + console.error("ERROR fetching file:", error) + return "" + } + }, + + createFile: async (fileId: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId }), + }) + return res.ok + }, + + renameFile: async ( + fileId: string, + newFileId: string, + data: string + ) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId, newFileId, data }), + }) + return res.ok + }, + + saveFile: async (fileId: string, data: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId, data }), + }) + return res.ok + }, + + deleteFile: async (fileId: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { + method: "DELETE", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId }), + }) + return res.ok + }, + + getProjectSize: async (id: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + return (await res.json()).size + } +} + +export default RemoteFileStorage \ No newline at end of file diff --git a/backend/server/src/fileoperations.ts b/backend/server/src/fileoperations.ts deleted file mode 100644 index 1157487..0000000 --- a/backend/server/src/fileoperations.ts +++ /dev/null @@ -1,170 +0,0 @@ -import * as dotenv from "dotenv" -import { R2Files, TFile, TFileData, TFolder } from "./types" - -dotenv.config() - -export const getSandboxFiles = async (id: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - const data: R2Files = await res.json() - - const paths = data.objects.map((obj) => obj.key) - const processedFiles = await processFiles(paths, id) - return processedFiles -} - -export const getFolder = async (folderId: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - const data: R2Files = await res.json() - - return data.objects.map((obj) => obj.key) -} - -const processFiles = async (paths: string[], id: string) => { - const root: TFolder = { id: "/", type: "folder", name: "/", children: [] } - const fileData: TFileData[] = [] - - paths.forEach((path) => { - const allParts = path.split("/") - if (allParts[1] !== id) { - return - } - - const parts = allParts.slice(2) - let current: TFolder = root - - for (let i = 0; i < parts.length; i++) { - const part = parts[i] - const isFile = i === parts.length - 1 && part.length - const existing = current.children.find((child) => child.name === part) - - if (existing) { - if (!isFile) { - current = existing as TFolder - } - } else { - if (isFile) { - const file: TFile = { id: path, type: "file", name: part } - current.children.push(file) - fileData.push({ id: path, data: "" }) - } else { - const folder: TFolder = { - // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css - id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, - type: "folder", - name: part, - children: [], - } - current.children.push(folder) - current = folder - } - } - } - }) - - await Promise.all( - fileData.map(async (file) => { - const data = await fetchFileContent(file.id) - file.data = data - }) - ) - - return { - files: root.children, - fileData, - } -} - -const fetchFileContent = async (fileId: string): Promise => { - try { - const fileRes = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - return await fileRes.text() - } catch (error) { - console.error("ERROR fetching file:", error) - return "" - } -} - -export const createFile = async (fileId: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId }), - }) - return res.ok -} - -export const renameFile = async ( - fileId: string, - newFileId: string, - data: string -) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId, newFileId, data }), - }) - return res.ok -} - -export const saveFile = async (fileId: string, data: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId, data }), - }) - return res.ok -} - -export const deleteFile = async (fileId: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { - method: "DELETE", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId }), - }) - return res.ok -} - -export const getProjectSize = async (id: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - return (await res.json()).size -} From 4221d7d09ad7b0aea1a33b3c4e7627f5c6efc6a4 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 16:41:26 -0600 Subject: [PATCH 09/10] chore: use fixed path for the project directory --- backend/server/src/FileManager.ts | 62 +++++++++++---------------- backend/server/src/TerminalManager.ts | 11 +---- backend/server/src/index.ts | 1 - 3 files changed, 27 insertions(+), 47 deletions(-) diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index d615c43..9928a75 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -34,13 +34,12 @@ const processFiles = async (paths: string[], id: string) => { } } else { if (isFile) { - const file: TFile = { id: path, type: "file", name: part } + const file: TFile = { id: `/${parts.join("/")}`, type: "file", name: part } current.children.push(file) - fileData.push({ id: path, data: "" }) + fileData.push({ id: `/${parts.join("/")}`, data: "" }) } else { const folder: TFolder = { - // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css - id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, + id: `/${parts.slice(0, i + 1).join("/")}`, type: "folder", name: part, children: [], @@ -54,7 +53,7 @@ const processFiles = async (paths: string[], id: string) => { await Promise.all( fileData.map(async (file) => { - const data = await RemoteFileStorage.fetchFileContent(file.id) + const data = await RemoteFileStorage.fetchFileContent(`projects/${id}${file.id}`) file.data = data }) ) @@ -75,7 +74,7 @@ export class FileManager { private sandbox: Sandbox public sandboxFiles: SandboxFiles private fileWatchers: WatchHandle[] = [] - private dirName = "/home/user" + private dirName = "/home/user/project" private refreshFileList: (files: SandboxFiles) => void // Constructor to initialize the FileManager @@ -90,14 +89,14 @@ export class FileManager { this.refreshFileList = refreshFileList } + private getRemoteFileId(localId: string): string { + return `projects/${this.sandboxId}${localId}` + } + // Initialize the FileManager async initialize() { this.sandboxFiles = await getSandboxFiles(this.sandboxId) - const projectDirectory = path.posix.join( - this.dirName, - "projects", - this.sandboxId - ) + const projectDirectory = this.dirName // Copy all files from the project to the container const promises = this.sandboxFiles.fileData.map(async (file) => { try { @@ -136,13 +135,8 @@ export class FileManager { // Change the owner of the project directory to user private async fixPermissions() { try { - const projectDirectory = path.posix.join( - this.dirName, - "projects", - this.sandboxId - ) await this.sandbox.commands.run( - `sudo chown -R user "${projectDirectory}"` + `sudo chown -R user "${this.dirName}"` ) } catch (e: any) { console.log("Failed to fix permissions: " + e) @@ -164,16 +158,10 @@ export class FileManager { // This is the absolute file path in the container const containerFilePath = path.posix.join(directory, event.name) - // This is the file path relative to the home directory - const sandboxFilePath = removeDirName( - containerFilePath, - this.dirName + "/" - ) - // This is the directory being watched relative to the home directory - const sandboxDirectory = removeDirName( - directory, - this.dirName + "/" - ) + // This is the file path relative to the project directory + const sandboxFilePath = removeDirName(containerFilePath, this.dirName) + // This is the directory being watched relative to the project directory + const sandboxDirectory = removeDirName(directory, this.dirName) // Helper function to find a folder by id function findFolderById( @@ -336,7 +324,7 @@ export class FileManager { // Get folder content async getFolder(folderId: string): Promise { - return RemoteFileStorage.getFolder(folderId) + return RemoteFileStorage.getFolder(this.getRemoteFileId(folderId)) } // Save file content @@ -346,7 +334,7 @@ export class FileManager { if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { throw new Error("File size too large. Please reduce the file size.") } - await RemoteFileStorage.saveFile(fileId, body) + await RemoteFileStorage.saveFile(this.getRemoteFileId(fileId), body) const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) if (!file) return file.data = body @@ -374,7 +362,7 @@ export class FileManager { fileData.id = newFileId file.id = newFileId - await RemoteFileStorage.renameFile(fileId, newFileId, fileData.data) + await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data) const newFiles = await getSandboxFiles(this.sandboxId) return newFiles.files } @@ -402,7 +390,7 @@ export class FileManager { throw new Error("Project size exceeded. Please delete some files.") } - const id = `projects/${this.sandboxId}/${name}` + const id = `/${name}` await this.sandbox.files.write(path.posix.join(this.dirName, id), "") await this.fixPermissions() @@ -418,14 +406,14 @@ export class FileManager { data: "", }) - await RemoteFileStorage.createFile(id) + await RemoteFileStorage.createFile(this.getRemoteFileId(id)) return true } // Create a new folder async createFolder(name: string): Promise { - const id = `projects/${this.sandboxId}/${name}` + const id = `/${name}` await this.sandbox.files.makeDir(path.posix.join(this.dirName, id)) } @@ -440,7 +428,7 @@ export class FileManager { await this.moveFileInContainer(fileId, newFileId) await this.fixPermissions() - await RemoteFileStorage.renameFile(fileId, newFileId, fileData.data) + await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data) fileData.id = newFileId file.id = newFileId @@ -456,7 +444,7 @@ export class FileManager { (f) => f.id !== fileId ) - await RemoteFileStorage.deleteFile(fileId) + await RemoteFileStorage.deleteFile(this.getRemoteFileId(fileId)) const newFiles = await getSandboxFiles(this.sandboxId) return newFiles.files @@ -464,7 +452,7 @@ export class FileManager { // Delete a folder async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> { - const files = await RemoteFileStorage.getFolder(folderId) + const files = await RemoteFileStorage.getFolder(this.getRemoteFileId(folderId)) await Promise.all( files.map(async (file) => { @@ -472,7 +460,7 @@ export class FileManager { this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter( (f) => f.id !== file ) - await RemoteFileStorage.deleteFile(file) + await RemoteFileStorage.deleteFile(this.getRemoteFileId(file)) }) ) diff --git a/backend/server/src/TerminalManager.ts b/backend/server/src/TerminalManager.ts index a9bf55c..b97aa6c 100644 --- a/backend/server/src/TerminalManager.ts +++ b/backend/server/src/TerminalManager.ts @@ -1,14 +1,11 @@ import { Sandbox } from "e2b" -import path from "path" import { Terminal } from "./Terminal" export class TerminalManager { - private sandboxId: string private sandbox: Sandbox private terminals: Record = {} - constructor(sandboxId: string, sandbox: Sandbox) { - this.sandboxId = sandboxId + constructor(sandbox: Sandbox) { this.sandbox = sandbox } @@ -27,11 +24,7 @@ export class TerminalManager { rows: 20, }) - const defaultDirectory = path.posix.join( - "/home/user", - "projects", - this.sandboxId - ) + const defaultDirectory = "/home/user/project" const defaultCommands = [ `cd "${defaultDirectory}"`, "export PS1='user> '", diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 2d9d42b..f69a303 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -226,7 +226,6 @@ io.on("connection", async (socket) => { ) await fileManagers[data.sandboxId].initialize() terminalManagers[data.sandboxId] = new TerminalManager( - data.sandboxId, containers[data.sandboxId] ) } From a459da6e6fea977d32cc72d1b5002f9e069c9121 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 18:42:44 -0600 Subject: [PATCH 10/10] chore: create separate functions to manage file structure and file data --- backend/server/src/FileManager.ts | 105 +++++++++++++++++++----------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index 9928a75..278d060 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -10,17 +10,12 @@ export type SandboxFiles = { fileData: TFileData[] } -const processFiles = async (paths: string[], id: string) => { +// Convert list of paths to the hierchical file structure used by the editor +function generateFileStructure(paths: string[]): (TFolder | TFile)[] { const root: TFolder = { id: "/", type: "folder", name: "/", children: [] } - const fileData: TFileData[] = [] paths.forEach((path) => { - const allParts = path.split("/") - if (allParts[1] !== id) { - return - } - - const parts = allParts.slice(2) + const parts = path.split("/") let current: TFolder = root for (let i = 0; i < parts.length; i++) { @@ -36,7 +31,6 @@ const processFiles = async (paths: string[], id: string) => { if (isFile) { const file: TFile = { id: `/${parts.join("/")}`, type: "file", name: part } current.children.push(file) - fileData.push({ id: `/${parts.join("/")}`, data: "" }) } else { const folder: TFolder = { id: `/${parts.slice(0, i + 1).join("/")}`, @@ -51,21 +45,7 @@ const processFiles = async (paths: string[], id: string) => { } }) - await Promise.all( - fileData.map(async (file) => { - const data = await RemoteFileStorage.fetchFileContent(`projects/${id}${file.id}`) - file.data = data - }) - ) - - return { - files: root.children, - fileData, - } -} - -const getSandboxFiles = async (id: string) => { - return await processFiles(await RemoteFileStorage.getSandboxPaths(id), id) + return root.children } // FileManager class to handle file operations in a sandbox @@ -89,14 +69,66 @@ export class FileManager { this.refreshFileList = refreshFileList } + // Fetch file data from list of paths + private async generateFileData(paths: string[]): Promise { + const fileData: TFileData[] = [] + + for (const path of paths) { + const parts = path.split("/") + const isFile = parts.length > 0 && parts[parts.length - 1].length > 0 + + if (isFile) { + const fileId = `/${parts.join("/")}` + const data = await RemoteFileStorage.fetchFileContent(`projects/${this.sandboxId}${fileId}`) + fileData.push({ id: fileId, data }) + } + } + + return fileData + } + + // Convert local file path to remote path private getRemoteFileId(localId: string): string { return `projects/${this.sandboxId}${localId}` } + // Convert remote file path to local file path + private getLocalFileId(remoteId: string): string | undefined { + const allParts = remoteId.split("/") + if (allParts[1] !== this.sandboxId) return undefined; + return allParts.slice(2).join("/") + } + + // Convert remote file paths to local file paths + private getLocalFileIds(remoteIds: string[]): string[] { + return remoteIds + .map(this.getLocalFileId.bind(this)) + .filter((id) => id !== undefined); + } + + // Download files from remote storage + private async updateFileData(): Promise { + const remotePaths = await RemoteFileStorage.getSandboxPaths(this.sandboxId) + const localPaths = this.getLocalFileIds(remotePaths) + this.sandboxFiles.fileData = await this.generateFileData(localPaths) + return this.sandboxFiles.fileData + } + + // Update file structure + private async updateFileStructure(): Promise<(TFolder | TFile)[]> { + const remotePaths = await RemoteFileStorage.getSandboxPaths(this.sandboxId) + const localPaths = this.getLocalFileIds(remotePaths) + this.sandboxFiles.files = generateFileStructure(localPaths) + return this.sandboxFiles.files + } + // Initialize the FileManager async initialize() { - this.sandboxFiles = await getSandboxFiles(this.sandboxId) - const projectDirectory = this.dirName + + // Download files from remote file storage + await this.updateFileStructure() + await this.updateFileData() + // Copy all files from the project to the container const promises = this.sandboxFiles.fileData.map(async (file) => { try { @@ -115,15 +147,15 @@ export class FileManager { // Make the logged in user the owner of all project files this.fixPermissions() - await this.watchDirectory(projectDirectory) - await this.watchSubdirectories(projectDirectory) + await this.watchDirectory(this.dirName) + await this.watchSubdirectories(this.dirName) } // Check if the given path is a directory - private async isDirectory(projectDirectory: string): Promise { + private async isDirectory(directoryPath: string): Promise { try { const result = await this.sandbox.commands.run( - `[ -d "${projectDirectory}" ] && echo "true" || echo "false"` + `[ -d "${directoryPath}" ] && echo "true" || echo "false"` ) return result.stdout.trim() === "true" } catch (e: any) { @@ -324,7 +356,8 @@ export class FileManager { // Get folder content async getFolder(folderId: string): Promise { - return RemoteFileStorage.getFolder(this.getRemoteFileId(folderId)) + const remotePaths = await RemoteFileStorage.getFolder(this.getRemoteFileId(folderId)) + return this.getLocalFileIds(remotePaths) } // Save file content @@ -363,8 +396,7 @@ export class FileManager { file.id = newFileId await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data) - const newFiles = await getSandboxFiles(this.sandboxId) - return newFiles.files + return this.updateFileStructure() } // Move a file within the container @@ -445,9 +477,7 @@ export class FileManager { ) await RemoteFileStorage.deleteFile(this.getRemoteFileId(fileId)) - - const newFiles = await getSandboxFiles(this.sandboxId) - return newFiles.files + return this.updateFileStructure() } // Delete a folder @@ -464,8 +494,7 @@ export class FileManager { }) ) - const newFiles = await getSandboxFiles(this.sandboxId) - return newFiles.files + return this.updateFileStructure() } // Close all file watchers