From 5a63ab7265454ae59548b6f50f4226dd732c9ba9 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 2 Nov 2024 13:27:11 -0600 Subject: [PATCH] feat: load project templates from custom E2B sandboxes instead of from Cloudflare --- backend/server/src/FileManager.ts | 11 +++++++-- backend/server/src/Sandbox.ts | 15 ++++++++---- backend/server/src/index.ts | 2 ++ backend/server/src/socketAuth.ts | 14 ++++++++++- backend/storage/src/index.ts | 29 +---------------------- frontend/components/editor/navbar/run.tsx | 4 ++-- 6 files changed, 37 insertions(+), 38 deletions(-) diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index 88d53f3..3d62918 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -140,6 +140,12 @@ export class FileManager { }) await Promise.all(promises) + // Reload file list from the container to include template files + const result = await this.sandbox.commands.run(`find "${this.dirName}" -type f`); // List all files recursively + const localPaths = result.stdout.split('\n').filter(path => path); // Split the output into an array and filter out empty strings + const relativePaths = localPaths.map(filePath => path.relative(this.dirName, filePath)); // Convert absolute paths to relative paths + this.files = generateFileStructure(relativePaths); + // Make the logged in user the owner of all project files this.fixPermissions() @@ -348,8 +354,9 @@ export class FileManager { // Get file content async getFile(fileId: string): Promise { - const file = this.fileData.find((f) => f.id === fileId) - return file?.data + const filePath = path.posix.join(this.dirName, fileId) + const fileContent = await this.sandbox.files.read(filePath) + return fileContent } // Get folder content diff --git a/backend/server/src/Sandbox.ts b/backend/server/src/Sandbox.ts index e7dd1bf..5de7754 100644 --- a/backend/server/src/Sandbox.ts +++ b/backend/server/src/Sandbox.ts @@ -13,9 +13,8 @@ import { } from "./ratelimit" import { SecureGitClient } from "./SecureGitClient" import { TerminalManager } from "./TerminalManager" -import { TFile, TFolder } from "./types" +import { TFile, TFileData, TFolder } from "./types" import { LockManager } from "./utils" -import { TFileData } from "./types" const lockManager = new LockManager() // Define a type for SocketHandler functions @@ -38,6 +37,7 @@ type ServerContext = { export class Sandbox { // Sandbox properties: sandboxId: string; + type: string; fileManager: FileManager | null; terminalManager: TerminalManager | null; container: E2BSandbox | null; @@ -46,9 +46,10 @@ export class Sandbox { gitClient: SecureGitClient | null; aiWorker: AIWorker; - constructor(sandboxId: string, { aiWorker, dokkuClient, gitClient }: ServerContext) { + constructor(sandboxId: string, type: string, { aiWorker, dokkuClient, gitClient }: ServerContext) { // Sandbox properties: this.sandboxId = sandboxId; + this.type = type; this.fileManager = null; this.terminalManager = null; this.container = null; @@ -69,8 +70,12 @@ export class Sandbox { console.log(`Found existing container ${this.sandboxId}`) } else { console.log("Creating container", this.sandboxId) - // Create a new container with a specified timeout - this.container = await E2BSandbox.create({ + // Create a new container with a specified template and timeout + const templateTypes = ["vanillajs", "reactjs", "nextjs", "streamlit"]; + const template = templateTypes.includes(this.type) + ? `gitwit-${this.type}` + : `base`; + this.container = await E2BSandbox.create(template, { timeoutMs: CONTAINER_TIMEOUT, }) } diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index cf95824..e1e5eda 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -96,6 +96,7 @@ io.on("connection", async (socket) => { userId: string sandboxId: string isOwner: boolean + type: string } // Register the connection @@ -111,6 +112,7 @@ io.on("connection", async (socket) => { // Create or retrieve the sandbox manager for the given sandbox ID const sandbox = sandboxes[data.sandboxId] ?? new Sandbox( data.sandboxId, + data.type, { aiWorker, dokkuClient, gitClient, } diff --git a/backend/server/src/socketAuth.ts b/backend/server/src/socketAuth.ts index 3bd83b1..030a89b 100644 --- a/backend/server/src/socketAuth.ts +++ b/backend/server/src/socketAuth.ts @@ -1,6 +1,6 @@ import { Socket } from "socket.io" import { z } from "zod" -import { User } from "./types" +import { Sandbox, User } from "./types" // Middleware for socket authentication export const socketAuth = async (socket: Socket, next: Function) => { @@ -33,6 +33,17 @@ export const socketAuth = async (socket: Socket, next: Function) => { ) const dbUserJSON = (await dbUser.json()) as User + // Fetch sandbox data from the database + const dbSandbox = await fetch( + `${process.env.DATABASE_WORKER_URL}/api/sandbox?id=${sandboxId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + const dbSandboxJSON = (await dbSandbox.json()) as Sandbox + // Check if user data was retrieved successfully if (!dbUserJSON) { next(new Error("DB error.")) @@ -56,6 +67,7 @@ export const socketAuth = async (socket: Socket, next: Function) => { userId, sandboxId: sandboxId, isOwner: sandbox !== undefined, + type: dbSandboxJSON.type } // Allow the connection diff --git a/backend/storage/src/index.ts b/backend/storage/src/index.ts index e7a7294..22def5d 100644 --- a/backend/storage/src/index.ts +++ b/backend/storage/src/index.ts @@ -1,4 +1,3 @@ -import pLimit from "p-limit" import { z } from "zod" export interface Env { @@ -136,33 +135,7 @@ export default { return success } else if (path === "/api/init" && method === "POST") { - const initSchema = z.object({ - sandboxId: z.string(), - type: z.string(), - }) - - const body = await request.json() - const { sandboxId, type } = initSchema.parse(body) - - console.log(`Copying template: ${type}`) - - // List all objects under the directory - const { objects } = await env.Templates.list({ prefix: type }) - - // Copy each object to the new directory with a 5 concurrency limit - const limit = pLimit(5) - await Promise.all( - objects.map(({ key }) => - limit(async () => { - const destinationKey = key.replace(type, `projects/${sandboxId}`) - const fileBody = await env.Templates.get(key).then( - (res) => res?.body ?? "" - ) - await env.R2.put(destinationKey, fileBody) - }) - ) - ) - + // This API path no longer does anything, because template files are stored in E2B sandbox templates. return success } else { return notFound diff --git a/frontend/components/editor/navbar/run.tsx b/frontend/components/editor/navbar/run.tsx index 2d11008..c6020aa 100644 --- a/frontend/components/editor/navbar/run.tsx +++ b/frontend/components/editor/navbar/run.tsx @@ -44,8 +44,8 @@ export default function RunButtonModal({ } else if (!isRunning && terminals.length < 4) { const command = sandboxData.type === "streamlit" - ? "pip install -r requirements.txt && streamlit run main.py --server.runOnSave true" - : "yarn install && yarn dev" + ? "./venv/bin/streamlit run main.py --server.runOnSave true" + : "npm run dev" try { // Create a new terminal with the appropriate command