From b561f1e96202b958b81c977ed1c6a06644c6bf5e Mon Sep 17 00:00:00 2001 From: James Murdza Date: Fri, 14 Jun 2024 11:57:32 -0400 Subject: [PATCH 01/29] chore: rename utils.ts to fileoperations.ts --- backend/server/src/fileoperations.ts | 177 ++++++++++++++++++++++++ backend/server/src/index.ts | 5 +- backend/server/src/utils.ts | 192 +++------------------------ 3 files changed, 199 insertions(+), 175 deletions(-) create mode 100644 backend/server/src/fileoperations.ts diff --git a/backend/server/src/fileoperations.ts b/backend/server/src/fileoperations.ts new file mode 100644 index 0000000..141363d --- /dev/null +++ b/backend/server/src/fileoperations.ts @@ -0,0 +1,177 @@ +import * as dotenv from "dotenv"; +import { + R2FileBody, + R2Files, + Sandbox, + TFile, + TFileData, + TFolder, +} from "./types"; + +dotenv.config(); + +export const getSandboxFiles = async (id: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ); + const data: R2Files = await res.json(); + + const paths = data.objects.map((obj) => obj.key); + const processedFiles = await processFiles(paths, id); + return processedFiles; +}; + +export const getFolder = async (folderId: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ); + const data: R2Files = await res.json(); + + return data.objects.map((obj) => obj.key); +}; + +const processFiles = async (paths: string[], id: string) => { + const root: TFolder = { id: "/", type: "folder", name: "/", children: [] }; + const fileData: TFileData[] = []; + + paths.forEach((path) => { + const allParts = path.split("/"); + if (allParts[1] !== id) { + return; + } + + const parts = allParts.slice(2); + let current: TFolder = root; + + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const isFile = i === parts.length - 1 && part.includes("."); + const existing = current.children.find((child) => child.name === part); + + if (existing) { + if (!isFile) { + current = existing as TFolder; + } + } else { + if (isFile) { + const file: TFile = { id: path, type: "file", name: part }; + current.children.push(file); + fileData.push({ id: path, data: "" }); + } else { + const folder: TFolder = { + // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css + id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, + type: "folder", + name: part, + children: [], + }; + current.children.push(folder); + current = folder; + } + } + } + }); + + await Promise.all( + fileData.map(async (file) => { + const data = await fetchFileContent(file.id); + file.data = data; + }) + ); + + return { + files: root.children, + fileData, + }; +}; + +const fetchFileContent = async (fileId: string): Promise => { + try { + const fileRes = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ); + return await fileRes.text(); + } catch (error) { + console.error("ERROR fetching file:", error); + return ""; + } +}; + +export const createFile = async (fileId: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId }), + }); + return res.ok; +}; + +export const renameFile = async ( + fileId: string, + newFileId: string, + data: string +) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId, newFileId, data }), + }); + return res.ok; +}; + +export const saveFile = async (fileId: string, data: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId, data }), + }); + return res.ok; +}; + +export const deleteFile = async (fileId: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { + method: "DELETE", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId }), + }); + return res.ok; +}; + +export const getProjectSize = async (id: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ); + return (await res.json()).size; +}; \ No newline at end of file diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 92c45df..2f63070 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -17,8 +17,9 @@ import { getSandboxFiles, renameFile, saveFile, -} from "./utils"; -import { IDisposable, IPty, spawn } from "node-pty"; +} from "./fileoperations"; +import { LockManager } from "./utils"; +import { Sandbox, Terminal } from "e2b"; import { MAX_BODY_SIZE, createFileRL, diff --git a/backend/server/src/utils.ts b/backend/server/src/utils.ts index 51e28f9..0aebb03 100644 --- a/backend/server/src/utils.ts +++ b/backend/server/src/utils.ts @@ -1,177 +1,23 @@ -import * as dotenv from "dotenv"; -import { - R2FileBody, - R2Files, - Sandbox, - TFile, - TFileData, - TFolder, -} from "./types"; +export class LockManager { + private locks: { [key: string]: Promise }; -dotenv.config(); - -export const getSandboxFiles = async (id: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ); - const data: R2Files = await res.json(); - - const paths = data.objects.map((obj) => obj.key); - const processedFiles = await processFiles(paths, id); - return processedFiles; -}; - -export const getFolder = async (folderId: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ); - const data: R2Files = await res.json(); - - return data.objects.map((obj) => obj.key); -}; - -const processFiles = async (paths: string[], id: string) => { - const root: TFolder = { id: "/", type: "folder", name: "/", children: [] }; - const fileData: TFileData[] = []; - - paths.forEach((path) => { - const allParts = path.split("/"); - if (allParts[1] !== id) { - return; - } - - const parts = allParts.slice(2); - let current: TFolder = root; - - for (let i = 0; i < parts.length; i++) { - const part = parts[i]; - const isFile = i === parts.length - 1 && part.includes("."); - const existing = current.children.find((child) => child.name === part); - - if (existing) { - if (!isFile) { - current = existing as TFolder; - } - } else { - if (isFile) { - const file: TFile = { id: path, type: "file", name: part }; - current.children.push(file); - fileData.push({ id: path, data: "" }); - } else { - const folder: TFolder = { - // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css - id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, - type: "folder", - name: part, - children: [], - }; - current.children.push(folder); - current = folder; - } - } - } - }); - - await Promise.all( - fileData.map(async (file) => { - const data = await fetchFileContent(file.id); - file.data = data; - }) - ); - - return { - files: root.children, - fileData, - }; -}; - -const fetchFileContent = async (fileId: string): Promise => { - try { - const fileRes = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ); - return await fileRes.text(); - } catch (error) { - console.error("ERROR fetching file:", error); - return ""; + constructor() { + this.locks = {}; } -}; -export const createFile = async (fileId: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId }), - }); - return res.ok; -}; - -export const renameFile = async ( - fileId: string, - newFileId: string, - data: string -) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId, newFileId, data }), - }); - return res.ok; -}; - -export const saveFile = async (fileId: string, data: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId, data }), - }); - return res.ok; -}; - -export const deleteFile = async (fileId: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { - method: "DELETE", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId }), - }); - return res.ok; -}; - -export const getProjectSize = async (id: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, + async acquireLock(key: string, task: () => Promise): Promise { + if (!this.locks[key]) { + this.locks[key] = new Promise(async (resolve, reject) => { + try { + const result = await task(); + resolve(result); + } catch (error) { + reject(error); + } finally { + delete this.locks[key]; + } + }); } - ); - return (await res.json()).size; -}; + return await this.locks[key]; + } +} \ No newline at end of file From e5b320d1c57c6402b3631677e61a7a90081f0e0c Mon Sep 17 00:00:00 2001 From: James Murdza Date: Fri, 14 Jun 2024 12:02:20 -0400 Subject: [PATCH 02/29] feat: replace node-pty with E2B sandboxes --- backend/server/package-lock.json | 131 +++++++++++++++++++++++++++---- backend/server/package.json | 2 +- backend/server/src/index.ts | 82 +++++++++++-------- 3 files changed, 166 insertions(+), 49 deletions(-) diff --git a/backend/server/package-lock.json b/backend/server/package-lock.json index e0262a2..8c7b2a0 100644 --- a/backend/server/package-lock.json +++ b/backend/server/package-lock.json @@ -12,8 +12,8 @@ "concurrently": "^8.2.2", "cors": "^2.8.5", "dotenv": "^16.4.5", + "e2b": "^0.16.1", "express": "^4.19.2", - "node-pty": "^1.0.0", "rate-limiter-flexible": "^5.0.3", "socket.io": "^4.7.5", "zod": "^3.22.4" @@ -369,6 +369,19 @@ "node": ">=8" } }, + "node_modules/bufferutil": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.8.tgz", + "integrity": "sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -662,6 +675,59 @@ "url": "https://dotenvx.com" } }, + "node_modules/e2b": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/e2b/-/e2b-0.16.1.tgz", + "integrity": "sha512-2L1R/REEB+EezD4Q4MmcXXNATjvCYov2lv/69+PY6V95+wl1PZblIMTYAe7USxX6P6sqANxNs+kXqZr6RvXkSw==", + "dependencies": { + "isomorphic-ws": "^5.0.0", + "normalize-path": "^3.0.0", + "openapi-typescript-fetch": "^1.1.3", + "path-browserify": "^1.0.1", + "platform": "^1.3.6", + "ws": "^8.15.1" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "bufferutil": "^4.0.8", + "utf-8-validate": "^6.0.3" + } + }, + "node_modules/e2b/node_modules/utf-8-validate": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-6.0.4.tgz", + "integrity": "sha512-xu9GQDeFp+eZ6LnCywXN/zBancWvOpUMzgjLPSjy4BRHSmTelvn2E0DG0o1sTiw5hkCKBHo8rwSKncfRfv2EEQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, + "node_modules/e2b/node_modules/ws": { + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1082,6 +1148,14 @@ "node": ">=0.12.0" } }, + "node_modules/isomorphic-ws": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-5.0.0.tgz", + "integrity": "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==", + "peerDependencies": { + "ws": "*" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -1173,11 +1247,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/nan": { - "version": "2.19.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz", - "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==" - }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -1186,13 +1255,15 @@ "node": ">= 0.6" } }, - "node_modules/node-pty": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-pty/-/node-pty-1.0.0.tgz", - "integrity": "sha512-wtBMWWS7dFZm/VgqElrTvtfMq4GzJ6+edFI0Y0zyzygUSZMgZdraDUMUhCIvkjhJjme15qWmbyJbtAx4ot4uZA==", - "hasInstallScript": true, - "dependencies": { - "nan": "^2.17.0" + "node_modules/node-gyp-build": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.1.tgz", + "integrity": "sha512-OSs33Z9yWr148JZcbZd5WiAXhh/n9z8TxQcdMhIOlpN9AhWpLfvVFO73+m77bBABQMaY9XSvIa+qk0jlI7Gcaw==", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" } }, "node_modules/nodemon": { @@ -1265,7 +1336,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -1297,6 +1367,15 @@ "node": ">= 0.8" } }, + "node_modules/openapi-typescript-fetch": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/openapi-typescript-fetch/-/openapi-typescript-fetch-1.1.3.tgz", + "integrity": "sha512-smLZPck4OkKMNExcw8jMgrMOGgVGx2N/s6DbKL2ftNl77g5HfoGpZGFy79RBzU/EkaO0OZpwBnslfdBfh7ZcWg==", + "engines": { + "node": ">= 12.0.0", + "npm": ">= 7.0.0" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1305,6 +1384,11 @@ "node": ">= 0.8" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -1322,6 +1406,11 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1835,6 +1924,20 @@ "node": ">= 0.8" } }, + "node_modules/utf-8-validate": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", + "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", + "hasInstallScript": true, + "optional": true, + "peer": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/backend/server/package.json b/backend/server/package.json index e0ac9c5..3b89d67 100644 --- a/backend/server/package.json +++ b/backend/server/package.json @@ -14,8 +14,8 @@ "concurrently": "^8.2.2", "cors": "^2.8.5", "dotenv": "^16.4.5", + "e2b": "^0.16.1", "express": "^4.19.2", - "node-pty": "^1.0.0", "rate-limiter-flexible": "^5.0.3", "socket.io": "^4.7.5", "zod": "^3.22.4" diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 2f63070..ebf8df4 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -44,9 +44,8 @@ const io = new Server(httpServer, { let inactivityTimeout: NodeJS.Timeout | null = null; let isOwnerConnected = false; -const terminals: { - [id: string]: { terminal: IPty; onData: IDisposable; onExit: IDisposable }; -} = {}; +const containers: Record = {}; +const terminals: Record = {}; const dirName = path.join(__dirname, ".."); @@ -101,6 +100,8 @@ io.use(async (socket, next) => { next(); }); +const lockManager = new LockManager(); + io.on("connection", async (socket) => { if (inactivityTimeout) clearTimeout(inactivityTimeout); @@ -119,6 +120,17 @@ io.on("connection", async (socket) => { } } + await lockManager.acquireLock(data.sandboxId, async () => { + try { + if (!containers[data.sandboxId]) { + containers[data.sandboxId] = await Sandbox.create(); + console.log("Created container ", data.sandboxId); + } + } catch (error) { + console.error("Error creating container ", data.sandboxId, error); + } + }); + const sandboxFiles = await getSandboxFiles(data.sandboxId); sandboxFiles.fileData.forEach((file) => { const filePath = path.join(dirName, file.id); @@ -320,41 +332,33 @@ io.on("connection", async (socket) => { callback(newFiles.files); }); - socket.on("createTerminal", (id: string, callback) => { + socket.on("createTerminal", async (id: string, callback) => { if (terminals[id] || Object.keys(terminals).length >= 4) { return; } - const pty = spawn(os.platform() === "win32" ? "cmd.exe" : "bash", [], { - name: "xterm", - cols: 100, - cwd: path.join(dirName, "projects", data.sandboxId), + await lockManager.acquireLock(data.sandboxId, async () => { + try { + terminals[id] = await containers[data.sandboxId].terminal.start({ + onData: (data: string) => { + io.emit("terminalResponse", { id, data }); + }, + size: { cols: 80, rows: 20 }, + onExit: () => console.log("Terminal exited", id), + }); + await terminals[id].sendData("export PS1='user> '\rclear\r"); + console.log("Created terminal", id); + } catch (error) { + console.error("Error creating terminal ", id, error); + } }); - const onData = pty.onData((data) => { - io.emit("terminalResponse", { - id, - data, - }); - }); - - const onExit = pty.onExit((code) => console.log("exit :(", code)); - - pty.write("export PS1='\\u > '\r"); - pty.write("clear\r"); - - terminals[id] = { - terminal: pty, - onData, - onExit, - }; - callback(); }); socket.on("resizeTerminal", (dimensions: { cols: number; rows: number }) => { Object.values(terminals).forEach((t) => { - t.terminal.resize(dimensions.cols, dimensions.rows); + t.resize(dimensions); }); }); @@ -364,19 +368,18 @@ io.on("connection", async (socket) => { } try { - terminals[id].terminal.write(data); + terminals[id].sendData(data); } catch (e) { console.log("Error writing to terminal", e); } }); - socket.on("closeTerminal", (id: string, callback) => { + socket.on("closeTerminal", async (id: string, callback) => { if (!terminals[id]) { return; } - terminals[id].onData.dispose(); - terminals[id].onExit.dispose(); + await terminals[id].kill(); delete terminals[id]; callback(); @@ -430,12 +433,23 @@ io.on("connection", async (socket) => { socket.on("disconnect", async () => { if (data.isOwner) { Object.entries(terminals).forEach((t) => { - const { terminal, onData, onExit } = t[1]; - onData.dispose(); - onExit.dispose(); + const terminal = t[1]; + terminal.kill(); delete terminals[t[0]]; }); + await lockManager.acquireLock(data.sandboxId, async () => { + try { + if (containers[data.sandboxId]) { + await containers[data.sandboxId].close(); + delete containers[data.sandboxId]; + console.log("Closed container", data.sandboxId); + } + } catch (error) { + console.error("Error closing container ", data.sandboxId, error); + } + }); + socket.broadcast.emit( "disableAccess", "The sandbox owner has disconnected." From 0df074924f396fccce558ac823052cbf614c5cad Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Thu, 13 Jun 2024 15:50:47 +0000 Subject: [PATCH 03/29] added debounced function in the editor --- frontend/components/editor/index.tsx | 90 +++++++++++++++++----------- frontend/lib/utils.ts | 10 ++++ 2 files changed, 65 insertions(+), 35 deletions(-) diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index e3f3030..16c1c5d 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -1,6 +1,6 @@ "use client" -import { useEffect, useRef, useState } from "react" +import { SetStateAction, useCallback, useEffect, useRef, useState } from "react" import monaco from "monaco-editor" import Editor, { BeforeMount, OnMount } from "@monaco-editor/react" import { io } from "socket.io-client" @@ -23,7 +23,7 @@ import Tab from "../ui/tab" import Sidebar from "./sidebar" import GenerateInput from "./generate" import { Sandbox, User, TFile, TFolder, TTab } from "@/lib/types" -import { addNew, processFileType, validateName } from "@/lib/utils" +import { addNew, processFileType, validateName, debounce } from "@/lib/utils" import { Cursors } from "./live/cursors" import { Terminal } from "@xterm/xterm" import DisableAccessModal from "./live/disableModal" @@ -290,26 +290,32 @@ export default function CodeEditor({ }, [decorations.options]) // Save file keybinding logic effect + const debouncedSaveData = useCallback( + debounce((value: string | undefined, activeFileId: string | undefined) => { + setTabs((prev) => + prev.map((tab) => + tab.id === activeFileId ? { ...tab, saved: true } : tab + ) + ); + console.log(`Saving file...${activeFileId}`); + socket.emit("saveFile", activeFileId, value); + }, Number(process.env.FILE_SAVE_DEBOUNCE_DELAY)||1000), + [socket] + ); + useEffect(() => { const down = (e: KeyboardEvent) => { if (e.key === "s" && (e.metaKey || e.ctrlKey)) { - e.preventDefault() - - setTabs((prev) => - prev.map((tab) => - tab.id === activeFileId ? { ...tab, saved: true } : tab - ) - ) - - socket.emit("saveFile", activeFileId, editorRef?.getValue()) + e.preventDefault(); + debouncedSaveData(editorRef?.getValue(), activeFileId); } - } - document.addEventListener("keydown", down) + }; + document.addEventListener("keydown", down); return () => { - document.removeEventListener("keydown", down) - } - }, [tabs, activeFileId]) + document.removeEventListener("keydown", down); + }; + }, [activeFileId, tabs, debouncedSaveData]); // Liveblocks live collaboration setup effect useEffect(() => { @@ -417,31 +423,44 @@ export default function CodeEditor({ // Helper functions for tabs: // Select file and load content - const selectFile = (tab: TTab) => { - if (tab.id === activeFileId) return - setGenerate((prev) => { - return { - ...prev, - show: false, - } - }) - const exists = tabs.find((t) => t.id === tab.id) + // Initialize debounced function once + const fileCache = useRef(new Map()); + // Debounced function to get file content + const debouncedGetFile = useCallback( + debounce((tabId, callback) => { + socket.emit('getFile', tabId, callback); + }, 300), // 300ms debounce delay, adjust as needed + [] + ); + + const selectFile = useCallback((tab: TTab) => { + if (tab.id === activeFileId) return; + + setGenerate((prev) => ({ ...prev, show: false })); + + const exists = tabs.find((t) => t.id === tab.id); setTabs((prev) => { if (exists) { - setActiveFileId(exists.id) - return prev + setActiveFileId(exists.id); + return prev; } - return [...prev, tab] - }) + return [...prev, tab]; + }); - socket.emit("getFile", tab.id, (response: string) => { - setActiveFileContent(response) - }) - setEditorLanguage(processFileType(tab.name)) - setActiveFileId(tab.id) - } + if (fileCache.current.has(tab.id)) { + setActiveFileContent(fileCache.current.get(tab.id)); + } else { + debouncedGetFile(tab.id, (response: SetStateAction) => { + fileCache.current.set(tab.id, response); + setActiveFileContent(response); + }); + } + + setEditorLanguage(processFileType(tab.name)); + setActiveFileId(tab.id); + }, [activeFileId, tabs, debouncedGetFile]); // Close tab and remove from tabs const closeTab = (id: string) => { @@ -772,3 +791,4 @@ export default function CodeEditor({ ) } + diff --git a/frontend/lib/utils.ts b/frontend/lib/utils.ts index c52d06b..85cc434 100644 --- a/frontend/lib/utils.ts +++ b/frontend/lib/utils.ts @@ -61,3 +61,13 @@ export function addNew( ]) } } + +export function debounce void>(func: T, wait: number): T { + let timeout: NodeJS.Timeout | null = null; + return function (...args: Parameters) { + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(() => func(...args), wait); + } as T; +} \ No newline at end of file From a0fb905a04d943da08c1da1cfc28addd18231243 Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Thu, 13 Jun 2024 17:47:46 +0000 Subject: [PATCH 04/29] fix: move socket connection to useRef --- frontend/components/editor/index.tsx | 69 +++++++++++++++------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 16c1c5d..dfd1809 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -3,7 +3,7 @@ import { SetStateAction, useCallback, useEffect, useRef, useState } from "react" import monaco from "monaco-editor" import Editor, { BeforeMount, OnMount } from "@monaco-editor/react" -import { io } from "socket.io-client" +import { Socket, io } from "socket.io-client" import { toast } from "sonner" import { useClerk } from "@clerk/nextjs" @@ -41,12 +41,16 @@ export default function CodeEditor({ sandboxData: Sandbox reactDefinitionFile: string }) { - const socket = io( - `http://localhost:${process.env.NEXT_PUBLIC_SERVER_PORT}?userId=${userData.id}&sandboxId=${sandboxData.id}`, - { - timeout: 2000, - } - ) + const socketRef = useRef(null); + + // Initialize socket connection if it doesn't exist + if (!socketRef.current) { + socketRef.current = io( + `http://localhost:${process.env.NEXT_PUBLIC_SERVER_PORT}?userId=${userData.id}&sandboxId=${sandboxData.id}`, + { + timeout: 2000, + } + );} const [isPreviewCollapsed, setIsPreviewCollapsed] = useState(true) const [disableAccess, setDisableAccess] = useState({ @@ -298,9 +302,10 @@ export default function CodeEditor({ ) ); console.log(`Saving file...${activeFileId}`); - socket.emit("saveFile", activeFileId, value); + console.log(`Saving file...${value}`); + socketRef.current?.emit("saveFile", activeFileId, value); }, Number(process.env.FILE_SAVE_DEBOUNCE_DELAY)||1000), - [socket] + [socketRef] ); useEffect(() => { @@ -364,10 +369,10 @@ export default function CodeEditor({ // Connection/disconnection effect useEffect(() => { - socket.connect() - + socketRef.current?.connect() + return () => { - socket.disconnect() + socketRef.current?.disconnect() } }, []) @@ -402,20 +407,20 @@ export default function CodeEditor({ }) } - socket.on("connect", onConnect) - socket.on("disconnect", onDisconnect) - socket.on("loaded", onLoadedEvent) - socket.on("rateLimit", onRateLimit) - socket.on("terminalResponse", onTerminalResponse) - socket.on("disableAccess", onDisableAccess) + socketRef.current?.on("connect", onConnect) + socketRef.current?.on("disconnect", onDisconnect) + socketRef.current?.on("loaded", onLoadedEvent) + socketRef.current?.on("rateLimit", onRateLimit) + socketRef.current?.on("terminalResponse", onTerminalResponse) + socketRef.current?.on("disableAccess", onDisableAccess) return () => { - socket.off("connect", onConnect) - socket.off("disconnect", onDisconnect) - socket.off("loaded", onLoadedEvent) - socket.off("rateLimit", onRateLimit) - socket.off("terminalResponse", onTerminalResponse) - socket.off("disableAccess", onDisableAccess) + socketRef.current?.off("connect", onConnect) + socketRef.current?.off("disconnect", onDisconnect) + socketRef.current?.off("loaded", onLoadedEvent) + socketRef.current?.off("rateLimit", onRateLimit) + socketRef.current?.off("terminalResponse", onTerminalResponse) + socketRef.current?.off("disableAccess", onDisableAccess) } // }, []); }, [terminals]) @@ -430,7 +435,7 @@ export default function CodeEditor({ // Debounced function to get file content const debouncedGetFile = useCallback( debounce((tabId, callback) => { - socket.emit('getFile', tabId, callback); + socketRef.current?.emit('getFile', tabId, callback); }, 300), // 300ms debounce delay, adjust as needed [] ); @@ -534,7 +539,7 @@ export default function CodeEditor({ return false } - socket.emit("renameFile", id, newName) + socketRef.current?.emit("renameFile", id, newName) setTabs((prev) => prev.map((tab) => (tab.id === id ? { ...tab, name: newName } : tab)) ) @@ -543,7 +548,7 @@ export default function CodeEditor({ } const handleDeleteFile = (file: TFile) => { - socket.emit("deleteFile", file.id, (response: (TFolder | TFile)[]) => { + socketRef.current?.emit("deleteFile", file.id, (response: (TFolder | TFile)[]) => { setFiles(response) }) closeTab(file.id) @@ -553,11 +558,11 @@ export default function CodeEditor({ setDeletingFolderId(folder.id) console.log("deleting folder", folder.id) - socket.emit("getFolder", folder.id, (response: string[]) => + socketRef.current?.emit("getFolder", folder.id, (response: string[]) => closeTabs(response) ) - socket.emit("deleteFolder", folder.id, (response: (TFolder | TFile)[]) => { + socketRef.current?.emit("deleteFolder", folder.id, (response: (TFolder | TFile)[]) => { setFiles(response) setDeletingFolderId("") }) @@ -584,7 +589,7 @@ export default function CodeEditor({ {generate.show && ai ? ( t.id === activeFileId)?.name ?? "", @@ -644,7 +649,7 @@ export default function CodeEditor({ handleRename={handleRename} handleDeleteFile={handleDeleteFile} handleDeleteFolder={handleDeleteFolder} - socket={socket} + socket={socketRef.current} setFiles={setFiles} addNew={(name, type) => addNew(name, type, setFiles, sandboxData)} deletingFolderId={deletingFolderId} @@ -776,7 +781,7 @@ export default function CodeEditor({ ) : (
From 7353e88567159bd6a33fca8c7540325ef658d5a3 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Fri, 14 Jun 2024 12:28:19 -0400 Subject: [PATCH 05/29] fix: wait until terminals are killed to close the container --- backend/server/src/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index ebf8df4..da51a46 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -432,11 +432,12 @@ io.on("connection", async (socket) => { socket.on("disconnect", async () => { if (data.isOwner) { - Object.entries(terminals).forEach((t) => { - const terminal = t[1]; - terminal.kill(); - delete terminals[t[0]]; - }); + await Promise.all( + Object.entries(terminals).map(async ([key, terminal]) => { + await terminal.kill(); + delete terminals[key]; + }) + ); await lockManager.acquireLock(data.sandboxId, async () => { try { From 869ae6c1485f7ea5bcc7f6a40329f7a74941e6da Mon Sep 17 00:00:00 2001 From: James Murdza Date: Fri, 14 Jun 2024 13:32:55 -0400 Subject: [PATCH 06/29] fix: ensure container remains open until all owner connections are closed --- backend/server/src/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index da51a46..22e4f56 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -45,6 +45,7 @@ let inactivityTimeout: NodeJS.Timeout | null = null; let isOwnerConnected = false; const containers: Record = {}; +const connections: Record = {}; const terminals: Record = {}; const dirName = path.join(__dirname, ".."); @@ -113,6 +114,7 @@ io.on("connection", async (socket) => { if (data.isOwner) { isOwnerConnected = true; + connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1; } else { if (!isOwnerConnected) { socket.emit("disableAccess", "The sandbox owner is not connected."); @@ -432,6 +434,10 @@ io.on("connection", async (socket) => { socket.on("disconnect", async () => { if (data.isOwner) { + connections[data.sandboxId]--; + } + + if (data.isOwner && connections[data.sandboxId] <= 0) { await Promise.all( Object.entries(terminals).map(async ([key, terminal]) => { await terminal.kill(); From 006c5cea6647ba04797b321eb5534f79f962da9d Mon Sep 17 00:00:00 2001 From: James Murdza Date: Fri, 14 Jun 2024 15:43:22 -0400 Subject: [PATCH 07/29] fix: sync files to container instead of local file system --- backend/server/src/index.ts | 61 ++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 22e4f56..0dc28a3 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -1,4 +1,3 @@ -import fs from "fs"; import os from "os"; import path from "path"; import cors from "cors"; @@ -19,7 +18,7 @@ import { saveFile, } from "./fileoperations"; import { LockManager } from "./utils"; -import { Sandbox, Terminal } from "e2b"; +import { Sandbox, Terminal, FilesystemManager } from "e2b"; import { MAX_BODY_SIZE, createFileRL, @@ -48,7 +47,13 @@ const containers: Record = {}; const connections: Record = {}; const terminals: Record = {}; -const dirName = path.join(__dirname, ".."); +const dirName = "/home/user"; + +const moveFile = async (filesystem: FilesystemManager, filePath: string, newFilePath: string) => { + const fileContents = await filesystem.readBytes(filePath) + await filesystem.writeBytes(newFilePath, fileContents); + await filesystem.remove(filePath); +} io.use(async (socket, next) => { const handshakeSchema = z.object({ @@ -134,12 +139,10 @@ io.on("connection", async (socket) => { }); const sandboxFiles = await getSandboxFiles(data.sandboxId); - sandboxFiles.fileData.forEach((file) => { + sandboxFiles.fileData.forEach(async (file) => { const filePath = path.join(dirName, file.id); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFile(filePath, file.data, function (err) { - if (err) throw err; - }); + await containers[data.sandboxId].filesystem.makeDir(path.dirname(filePath)); + await containers[data.sandboxId].filesystem.write(filePath, file.data); }); socket.emit("loaded", sandboxFiles.files); @@ -173,9 +176,7 @@ io.on("connection", async (socket) => { if (!file) return; file.data = body; - fs.writeFile(path.join(dirName, file.id), body, function (err) { - if (err) throw err; - }); + await containers[data.sandboxId].filesystem.write(path.join(dirName, file.id), body); await saveFile(fileId, body); } catch (e) { io.emit("rateLimit", "Rate limited: file saving. Please slow down."); @@ -189,13 +190,11 @@ io.on("connection", async (socket) => { const parts = fileId.split("/"); const newFileId = folderId + "/" + parts.pop(); - fs.rename( + await moveFile( + containers[data.sandboxId].filesystem, path.join(dirName, fileId), - path.join(dirName, newFileId), - function (err) { - if (err) throw err; - } - ); + path.join(dirName, newFileId) + ) file.id = newFileId; @@ -221,9 +220,7 @@ io.on("connection", async (socket) => { const id = `projects/${data.sandboxId}/${name}`; - fs.writeFile(path.join(dirName, id), "", function (err) { - if (err) throw err; - }); + await containers[data.sandboxId].filesystem.write(path.join(dirName, id), ""); sandboxFiles.files.push({ id, @@ -250,9 +247,7 @@ io.on("connection", async (socket) => { const id = `projects/${data.sandboxId}/${name}`; - fs.mkdir(path.join(dirName, id), { recursive: true }, function (err) { - if (err) throw err; - }); + await containers[data.sandboxId].filesystem.makeDir(path.join(dirName, id)); callback(); } catch (e) { @@ -272,13 +267,12 @@ io.on("connection", async (socket) => { const newFileId = parts.slice(0, parts.length - 1).join("/") + "/" + newName; - fs.rename( + + await moveFile( + containers[data.sandboxId].filesystem, path.join(dirName, fileId), - path.join(dirName, newFileId), - function (err) { - if (err) throw err; - } - ); + path.join(dirName, newFileId) + ) await renameFile(fileId, newFileId, file.data); } catch (e) { io.emit("rateLimit", "Rate limited: file renaming. Please slow down."); @@ -292,9 +286,7 @@ io.on("connection", async (socket) => { const file = sandboxFiles.fileData.find((f) => f.id === fileId); if (!file) return; - fs.unlink(path.join(dirName, fileId), function (err) { - if (err) throw err; - }); + await containers[data.sandboxId].filesystem.remove(path.join(dirName, fileId)); sandboxFiles.fileData = sandboxFiles.fileData.filter( (f) => f.id !== fileId ); @@ -317,9 +309,7 @@ io.on("connection", async (socket) => { await Promise.all( files.map(async (file) => { - fs.unlink(path.join(dirName, file), function (err) { - if (err) throw err; - }); + await containers[data.sandboxId].filesystem.remove(path.join(dirName, file)); sandboxFiles.fileData = sandboxFiles.fileData.filter( (f) => f.id !== file @@ -348,6 +338,7 @@ io.on("connection", async (socket) => { size: { cols: 80, rows: 20 }, onExit: () => console.log("Terminal exited", id), }); + await terminals[id].sendData(`cd "${path.join(dirName, "projects", data.sandboxId)}"\r`) await terminals[id].sendData("export PS1='user> '\rclear\r"); console.log("Created terminal", id); } catch (error) { From 687416e6e9c2fc7766c563cdda0236e904713c7c Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 15 Jun 2024 21:00:43 -0400 Subject: [PATCH 08/29] fix: set project file permissions so that they belong to the terminal user --- backend/server/src/index.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 0dc28a3..3b0b237 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -138,12 +138,20 @@ io.on("connection", async (socket) => { } }); + // Change the owner of the project directory to user + const fixPermissions = async () => { + await containers[data.sandboxId].process.startAndWait( + `sudo chown -R user "${path.join(dirName, "projects", data.sandboxId)}"` + ); + } + const sandboxFiles = await getSandboxFiles(data.sandboxId); sandboxFiles.fileData.forEach(async (file) => { const filePath = path.join(dirName, file.id); await containers[data.sandboxId].filesystem.makeDir(path.dirname(filePath)); await containers[data.sandboxId].filesystem.write(filePath, file.data); }); + fixPermissions(); socket.emit("loaded", sandboxFiles.files); @@ -177,6 +185,7 @@ io.on("connection", async (socket) => { file.data = body; await containers[data.sandboxId].filesystem.write(path.join(dirName, file.id), body); + fixPermissions(); await saveFile(fileId, body); } catch (e) { io.emit("rateLimit", "Rate limited: file saving. Please slow down."); @@ -195,6 +204,7 @@ io.on("connection", async (socket) => { path.join(dirName, fileId), path.join(dirName, newFileId) ) + fixPermissions(); file.id = newFileId; @@ -221,6 +231,7 @@ io.on("connection", async (socket) => { const id = `projects/${data.sandboxId}/${name}`; await containers[data.sandboxId].filesystem.write(path.join(dirName, id), ""); + fixPermissions(); sandboxFiles.files.push({ id, @@ -273,6 +284,7 @@ io.on("connection", async (socket) => { path.join(dirName, fileId), path.join(dirName, newFileId) ) + fixPermissions(); await renameFile(fileId, newFileId, file.data); } catch (e) { io.emit("rateLimit", "Rate limited: file renaming. Please slow down."); From 9ec59bc7811d2d519de01d619e895ca274c4b775 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 15 Jun 2024 21:04:33 -0400 Subject: [PATCH 09/29] fix: use the container URL for the preview panel --- backend/server/src/index.ts | 1 + frontend/components/editor/index.tsx | 6 ++++++ frontend/components/editor/preview/index.tsx | 6 ++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 3b0b237..2cb24c6 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -132,6 +132,7 @@ io.on("connection", async (socket) => { if (!containers[data.sandboxId]) { containers[data.sandboxId] = await Sandbox.create(); console.log("Created container ", data.sandboxId); + io.emit("previewURL", "https://" + containers[data.sandboxId].getHostname(5173)); } } catch (error) { console.error("Error creating container ", data.sandboxId, error); diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index dfd1809..2b0d16c 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -94,6 +94,9 @@ export default function CodeEditor({ }[] >([]) + // Preview state + const [previewURL, setPreviewURL] = useState(""); + const isOwner = sandboxData.userId === userData.id const clerk = useClerk() @@ -413,6 +416,7 @@ export default function CodeEditor({ socketRef.current?.on("rateLimit", onRateLimit) socketRef.current?.on("terminalResponse", onTerminalResponse) socketRef.current?.on("disableAccess", onDisableAccess) + socketRef.current?.on("previewURL", setPreviewURL) return () => { socketRef.current?.off("connect", onConnect) @@ -421,6 +425,7 @@ export default function CodeEditor({ socketRef.current?.off("rateLimit", onRateLimit) socketRef.current?.off("terminalResponse", onTerminalResponse) socketRef.current?.off("disableAccess", onDisableAccess) + socketRef.current?.off("previewURL", setPreviewURL) } // }, []); }, [terminals]) @@ -769,6 +774,7 @@ export default function CodeEditor({ previewPanelRef.current?.expand() setIsPreviewCollapsed(false) }} + src={previewURL} /> diff --git a/frontend/components/editor/preview/index.tsx b/frontend/components/editor/preview/index.tsx index 0544a12..a400d14 100644 --- a/frontend/components/editor/preview/index.tsx +++ b/frontend/components/editor/preview/index.tsx @@ -15,9 +15,11 @@ import { toast } from "sonner" export default function PreviewWindow({ collapsed, open, + src }: { collapsed: boolean open: () => void + src: string }) { const ref = useRef(null) const [iframeKey, setIframeKey] = useState(0) @@ -45,7 +47,7 @@ export default function PreviewWindow({ { - navigator.clipboard.writeText(`http://localhost:5173`) + navigator.clipboard.writeText(src) toast.info("Copied preview link to clipboard") }} > @@ -73,7 +75,7 @@ export default function PreviewWindow({ ref={ref} width={"100%"} height={"100%"} - src={`http://localhost:5173`} + src={src} />
)} From 97c8598717871b270519f4d6950998212e5cc8c5 Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Sun, 16 Jun 2024 19:53:02 -0400 Subject: [PATCH 10/29] fix: count only the current user's sandboxes towards the limit --- backend/database/src/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/database/src/index.ts b/backend/database/src/index.ts index 0d2721f..4eb9180 100644 --- a/backend/database/src/index.ts +++ b/backend/database/src/index.ts @@ -110,8 +110,13 @@ export default { const body = await request.json() const { type, name, userId, visibility } = initSchema.parse(body) - const allSandboxes = await db.select().from(sandbox).all() - if (allSandboxes.length >= 8) { + const userSandboxes = await db + .select() + .from(sandbox) + .where(eq(sandbox.userId, userId)) + .all() + + if (userSandboxes.length >= 8) { return new Response("You reached the maximum # of sandboxes.", { status: 400, }) From ed709210e31bf32d4e7587b9f65aad726bb4d374 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sun, 16 Jun 2024 20:01:00 -0400 Subject: [PATCH 11/29] fix: remove hardcoded reference to localhost --- frontend/components/editor/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 2b0d16c..d88c2e8 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -46,7 +46,7 @@ export default function CodeEditor({ // Initialize socket connection if it doesn't exist if (!socketRef.current) { socketRef.current = io( - `http://localhost:${process.env.NEXT_PUBLIC_SERVER_PORT}?userId=${userData.id}&sandboxId=${sandboxData.id}`, + `${window.location.protocol}//${window.location.hostname}:${process.env.NEXT_PUBLIC_SERVER_PORT}?userId=${userData.id}&sandboxId=${sandboxData.id}`, { timeout: 2000, } From c262fb2a31c4b0d5439aad6fb04555d27256d463 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Tue, 18 Jun 2024 19:40:56 -0400 Subject: [PATCH 12/29] fix: add error handling to the backend --- backend/server/src/index.ts | 820 +++++++++++++++------------ frontend/components/editor/index.tsx | 6 +- 2 files changed, 468 insertions(+), 358 deletions(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 2cb24c6..13ce955 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -49,11 +49,15 @@ const terminals: Record = {}; const dirName = "/home/user"; -const moveFile = async (filesystem: FilesystemManager, filePath: string, newFilePath: string) => { - const fileContents = await filesystem.readBytes(filePath) +const moveFile = async ( + filesystem: FilesystemManager, + filePath: string, + newFilePath: string +) => { + const fileContents = await filesystem.readBytes(filePath); await filesystem.writeBytes(newFilePath, fileContents); await filesystem.remove(filePath); -} +}; io.use(async (socket, next) => { const handshakeSchema = z.object({ @@ -109,381 +113,487 @@ io.use(async (socket, next) => { const lockManager = new LockManager(); io.on("connection", async (socket) => { - if (inactivityTimeout) clearTimeout(inactivityTimeout); + try { + if (inactivityTimeout) clearTimeout(inactivityTimeout); - const data = socket.data as { - userId: string; - sandboxId: string; - isOwner: boolean; - }; + const data = socket.data as { + userId: string; + sandboxId: string; + isOwner: boolean; + }; - if (data.isOwner) { - isOwnerConnected = true; - connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1; - } else { - if (!isOwnerConnected) { - socket.emit("disableAccess", "The sandbox owner is not connected."); - return; - } - } - - await lockManager.acquireLock(data.sandboxId, async () => { - try { - if (!containers[data.sandboxId]) { - containers[data.sandboxId] = await Sandbox.create(); - console.log("Created container ", data.sandboxId); - io.emit("previewURL", "https://" + containers[data.sandboxId].getHostname(5173)); - } - } catch (error) { - console.error("Error creating container ", data.sandboxId, error); - } - }); - - // Change the owner of the project directory to user - const fixPermissions = async () => { - await containers[data.sandboxId].process.startAndWait( - `sudo chown -R user "${path.join(dirName, "projects", data.sandboxId)}"` - ); - } - - const sandboxFiles = await getSandboxFiles(data.sandboxId); - sandboxFiles.fileData.forEach(async (file) => { - const filePath = path.join(dirName, file.id); - await containers[data.sandboxId].filesystem.makeDir(path.dirname(filePath)); - await containers[data.sandboxId].filesystem.write(filePath, file.data); - }); - fixPermissions(); - - socket.emit("loaded", sandboxFiles.files); - - socket.on("getFile", (fileId: string, callback) => { - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - - callback(file.data); - }); - - socket.on("getFolder", async (folderId: string, callback) => { - const files = await getFolder(folderId); - callback(files); - }); - - // todo: send diffs + debounce for efficiency - socket.on("saveFile", async (fileId: string, body: string) => { - try { - await saveFileRL.consume(data.userId, 1); - - if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { - socket.emit( - "rateLimit", - "Rate limited: file size too large. Please reduce the file size." - ); + if (data.isOwner) { + isOwnerConnected = true; + connections[data.sandboxId] = (connections[data.sandboxId] ?? 0) + 1; + } else { + if (!isOwnerConnected) { + socket.emit("disableAccess", "The sandbox owner is not connected."); return; } - - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - file.data = body; - - await containers[data.sandboxId].filesystem.write(path.join(dirName, file.id), body); - fixPermissions(); - await saveFile(fileId, body); - } catch (e) { - io.emit("rateLimit", "Rate limited: file saving. Please slow down."); - } - }); - - socket.on("moveFile", async (fileId: string, folderId: string, callback) => { - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - - const parts = fileId.split("/"); - const newFileId = folderId + "/" + parts.pop(); - - await moveFile( - containers[data.sandboxId].filesystem, - path.join(dirName, fileId), - path.join(dirName, newFileId) - ) - fixPermissions(); - - file.id = newFileId; - - await renameFile(fileId, newFileId, file.data); - const newFiles = await getSandboxFiles(data.sandboxId); - - callback(newFiles.files); - }); - - socket.on("createFile", async (name: string, callback) => { - try { - const size: number = await getProjectSize(data.sandboxId); - // limit is 200mb - if (size > 200 * 1024 * 1024) { - io.emit( - "rateLimit", - "Rate limited: project size exceeded. Please delete some files." - ); - callback({ success: false }); - } - - await createFileRL.consume(data.userId, 1); - - const id = `projects/${data.sandboxId}/${name}`; - - await containers[data.sandboxId].filesystem.write(path.join(dirName, id), ""); - fixPermissions(); - - sandboxFiles.files.push({ - id, - name, - type: "file", - }); - - sandboxFiles.fileData.push({ - id, - data: "", - }); - - await createFile(id); - - callback({ success: true }); - } catch (e) { - io.emit("rateLimit", "Rate limited: file creation. Please slow down."); - } - }); - - socket.on("createFolder", async (name: string, callback) => { - try { - await createFolderRL.consume(data.userId, 1); - - const id = `projects/${data.sandboxId}/${name}`; - - await containers[data.sandboxId].filesystem.makeDir(path.join(dirName, id)); - - callback(); - } catch (e) { - io.emit("rateLimit", "Rate limited: folder creation. Please slow down."); - } - }); - - socket.on("renameFile", async (fileId: string, newName: string) => { - try { - await renameFileRL.consume(data.userId, 1); - - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - file.id = newName; - - const parts = fileId.split("/"); - const newFileId = - parts.slice(0, parts.length - 1).join("/") + "/" + newName; - - - await moveFile( - containers[data.sandboxId].filesystem, - path.join(dirName, fileId), - path.join(dirName, newFileId) - ) - fixPermissions(); - await renameFile(fileId, newFileId, file.data); - } catch (e) { - io.emit("rateLimit", "Rate limited: file renaming. Please slow down."); - return; - } - }); - - socket.on("deleteFile", async (fileId: string, callback) => { - try { - await deleteFileRL.consume(data.userId, 1); - const file = sandboxFiles.fileData.find((f) => f.id === fileId); - if (!file) return; - - await containers[data.sandboxId].filesystem.remove(path.join(dirName, fileId)); - sandboxFiles.fileData = sandboxFiles.fileData.filter( - (f) => f.id !== fileId - ); - - await deleteFile(fileId); - - const newFiles = await getSandboxFiles(data.sandboxId); - callback(newFiles.files); - } catch (e) { - io.emit("rateLimit", "Rate limited: file deletion. Please slow down."); - } - }); - - // todo - // socket.on("renameFolder", async (folderId: string, newName: string) => { - // }); - - socket.on("deleteFolder", async (folderId: string, callback) => { - const files = await getFolder(folderId); - - await Promise.all( - files.map(async (file) => { - await containers[data.sandboxId].filesystem.remove(path.join(dirName, file)); - - sandboxFiles.fileData = sandboxFiles.fileData.filter( - (f) => f.id !== file - ); - - await deleteFile(file); - }) - ); - - const newFiles = await getSandboxFiles(data.sandboxId); - - callback(newFiles.files); - }); - - socket.on("createTerminal", async (id: string, callback) => { - if (terminals[id] || Object.keys(terminals).length >= 4) { - return; } await lockManager.acquireLock(data.sandboxId, async () => { try { - terminals[id] = await containers[data.sandboxId].terminal.start({ - onData: (data: string) => { - io.emit("terminalResponse", { id, data }); - }, - size: { cols: 80, rows: 20 }, - onExit: () => console.log("Terminal exited", id), - }); - await terminals[id].sendData(`cd "${path.join(dirName, "projects", data.sandboxId)}"\r`) - await terminals[id].sendData("export PS1='user> '\rclear\r"); - console.log("Created terminal", id); - } catch (error) { - console.error("Error creating terminal ", id, error); + if (!containers[data.sandboxId]) { + containers[data.sandboxId] = await Sandbox.create(); + console.log("Created container ", data.sandboxId); + io.emit( + "previewURL", + "https://" + containers[data.sandboxId].getHostname(5173) + ); + } + } catch (e: any) { + console.error(`Error creating container ${data.sandboxId}:`, e); + io.emit("error", `Error: container creation. ${e.message ?? e}`); } }); - callback(); - }); + // Change the owner of the project directory to user + const fixPermissions = async () => { + await containers[data.sandboxId].process.startAndWait( + `sudo chown -R user "${path.join(dirName, "projects", data.sandboxId)}"` + ); + }; - socket.on("resizeTerminal", (dimensions: { cols: number; rows: number }) => { - Object.values(terminals).forEach((t) => { - t.resize(dimensions); + const sandboxFiles = await getSandboxFiles(data.sandboxId); + sandboxFiles.fileData.forEach(async (file) => { + const filePath = path.join(dirName, file.id); + await containers[data.sandboxId].filesystem.makeDir( + path.dirname(filePath) + ); + await containers[data.sandboxId].filesystem.write(filePath, file.data); }); - }); + fixPermissions(); - socket.on("terminalData", (id: string, data: string) => { - if (!terminals[id]) { - return; - } + socket.emit("loaded", sandboxFiles.files); - try { - terminals[id].sendData(data); - } catch (e) { - console.log("Error writing to terminal", e); - } - }); + socket.on("getFile", (fileId: string, callback) => { + console.log(fileId); + try { + const file = sandboxFiles.fileData.find((f) => f.id === fileId); + if (!file) return; - socket.on("closeTerminal", async (id: string, callback) => { - if (!terminals[id]) { - return; - } + callback(file.data); + } catch (e: any) { + console.error("Error getting file:", e); + io.emit("error", `Error: get file. ${e.message ?? e}`); + } + }); - await terminals[id].kill(); - delete terminals[id]; + socket.on("getFolder", async (folderId: string, callback) => { + try { + const files = await getFolder(folderId); + callback(files); + } catch (e: any) { + console.error("Error getting folder:", e); + io.emit("error", `Error: get folder. ${e.message ?? e}`); + } + }); - callback(); - }); - - socket.on( - "generateCode", - async ( - fileName: string, - code: string, - line: number, - instructions: string, - callback - ) => { - const fetchPromise = fetch( - `${process.env.DATABASE_WORKER_URL}/api/sandbox/generate`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ - userId: data.userId, - }), + // todo: send diffs + debounce for efficiency + socket.on("saveFile", async (fileId: string, body: string) => { + try { + if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { + socket.emit( + "error", + "Error: file size too large. Please reduce the file size." + ); + return; } - ); - - // Generate code from cloudflare workers AI - const generateCodePromise = fetch( - `${process.env.AI_WORKER_URL}/api?fileName=${fileName}&code=${code}&line=${line}&instructions=${instructions}`, - { - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.CF_AI_KEY}`, - }, - } - ); - - const [fetchResponse, generateCodeResponse] = await Promise.all([ - fetchPromise, - generateCodePromise, - ]); - - const json = await generateCodeResponse.json(); - - callback({ response: json.response, success: true }); - } - ); - - socket.on("disconnect", async () => { - if (data.isOwner) { - connections[data.sandboxId]--; - } - - if (data.isOwner && connections[data.sandboxId] <= 0) { - await Promise.all( - Object.entries(terminals).map(async ([key, terminal]) => { - await terminal.kill(); - delete terminals[key]; - }) - ); - - await lockManager.acquireLock(data.sandboxId, async () => { try { - if (containers[data.sandboxId]) { - await containers[data.sandboxId].close(); - delete containers[data.sandboxId]; - console.log("Closed container", data.sandboxId); - } - } catch (error) { - console.error("Error closing container ", data.sandboxId, error); + await saveFileRL.consume(data.userId, 1); + await saveFile(fileId, body); + } catch (e) { + io.emit("error", "Rate limited: file saving. Please slow down."); + return; } - }); - socket.broadcast.emit( - "disableAccess", - "The sandbox owner has disconnected." - ); - } + const file = sandboxFiles.fileData.find((f) => f.id === fileId); + if (!file) return; + file.data = body; - // const sockets = await io.fetchSockets(); - // if (inactivityTimeout) { - // clearTimeout(inactivityTimeout); - // } - // if (sockets.length === 0) { - // console.log("STARTING TIMER"); - // inactivityTimeout = setTimeout(() => { - // io.fetchSockets().then(async (sockets) => { - // if (sockets.length === 0) { - // console.log("Server stopped", res); - // } - // }); - // }, 20000); - // } else { - // console.log("number of sockets", sockets.length); - // } - }); + await containers[data.sandboxId].filesystem.write( + path.join(dirName, file.id), + body + ); + fixPermissions(); + } catch (e: any) { + console.error("Error saving file:", e); + io.emit("error", `Error: file saving. ${e.message ?? e}`); + } + }); + + socket.on( + "moveFile", + async (fileId: string, folderId: string, callback) => { + try { + const file = sandboxFiles.fileData.find((f) => f.id === fileId); + if (!file) return; + + const parts = fileId.split("/"); + const newFileId = folderId + "/" + parts.pop(); + + await moveFile( + containers[data.sandboxId].filesystem, + path.join(dirName, fileId), + path.join(dirName, newFileId) + ); + fixPermissions(); + + file.id = newFileId; + + await renameFile(fileId, newFileId, file.data); + const newFiles = await getSandboxFiles(data.sandboxId); + callback(newFiles.files); + } catch (e: any) { + console.error("Error moving file:", e); + io.emit("error", `Error: file moving. ${e.message ?? e}`); + } + } + ); + + socket.on("createFile", async (name: string, callback) => { + try { + const size: number = await getProjectSize(data.sandboxId); + // limit is 200mb + if (size > 200 * 1024 * 1024) { + io.emit( + "error", + "Rate limited: project size exceeded. Please delete some files." + ); + callback({ success: false }); + return; + } + + try { + await createFileRL.consume(data.userId, 1); + } catch (e) { + io.emit("error", "Rate limited: file creation. Please slow down."); + return; + } + + const id = `projects/${data.sandboxId}/${name}`; + + await containers[data.sandboxId].filesystem.write( + path.join(dirName, id), + "" + ); + fixPermissions(); + + sandboxFiles.files.push({ + id, + name, + type: "file", + }); + + sandboxFiles.fileData.push({ + id, + data: "", + }); + + await createFile(id); + + callback({ success: true }); + } catch (e: any) { + console.error("Error creating file:", e); + io.emit("error", `Error: file creation. ${e.message ?? e}`); + } + }); + + socket.on("createFolder", async (name: string, callback) => { + try { + try { + await createFolderRL.consume(data.userId, 1); + } catch (e) { + io.emit("error", "Rate limited: folder creation. Please slow down."); + return; + } + + const id = `projects/${data.sandboxId}/${name}`; + + await containers[data.sandboxId].filesystem.makeDir( + path.join(dirName, id) + ); + + callback(); + } catch (e: any) { + console.error("Error creating folder:", e); + io.emit("error", `Error: folder creation. ${e.message ?? e}`); + } + }); + + socket.on("renameFile", async (fileId: string, newName: string) => { + try { + try { + await renameFileRL.consume(data.userId, 1); + } catch (e) { + io.emit("error", "Rate limited: file renaming. Please slow down."); + return; + } + + const file = sandboxFiles.fileData.find((f) => f.id === fileId); + if (!file) return; + file.id = newName; + + const parts = fileId.split("/"); + const newFileId = + parts.slice(0, parts.length - 1).join("/") + "/" + newName; + + await moveFile( + containers[data.sandboxId].filesystem, + path.join(dirName, fileId), + path.join(dirName, newFileId) + ); + fixPermissions(); + await renameFile(fileId, newFileId, file.data); + } catch (e: any) { + console.error("Error renaming folder:", e); + io.emit("error", `Error: folder renaming. ${e.message ?? e}`); + } + }); + + socket.on("deleteFile", async (fileId: string, callback) => { + try { + try { + await deleteFileRL.consume(data.userId, 1); + } catch (e) { + io.emit("error", "Rate limited: file deletion. Please slow down."); + } + + const file = sandboxFiles.fileData.find((f) => f.id === fileId); + if (!file) return; + + await containers[data.sandboxId].filesystem.remove( + path.join(dirName, fileId) + ); + sandboxFiles.fileData = sandboxFiles.fileData.filter( + (f) => f.id !== fileId + ); + + await deleteFile(fileId); + + const newFiles = await getSandboxFiles(data.sandboxId); + callback(newFiles.files); + } catch (e: any) { + console.error("Error deleting file:", e); + io.emit("error", `Error: file deletion. ${e.message ?? e}`); + } + }); + + // todo + // socket.on("renameFolder", async (folderId: string, newName: string) => { + // }); + + socket.on("deleteFolder", async (folderId: string, callback) => { + try { + const files = await getFolder(folderId); + + await Promise.all( + files.map(async (file) => { + await containers[data.sandboxId].filesystem.remove( + path.join(dirName, file) + ); + + sandboxFiles.fileData = sandboxFiles.fileData.filter( + (f) => f.id !== file + ); + + await deleteFile(file); + }) + ); + + const newFiles = await getSandboxFiles(data.sandboxId); + + callback(newFiles.files); + } catch (e: any) { + console.error("Error deleting folder:", e); + io.emit("error", `Error: folder deletion. ${e.message ?? e}`); + } + }); + + socket.on("createTerminal", async (id: string, callback) => { + try { + if (terminals[id] || Object.keys(terminals).length >= 4) { + return; + } + + await lockManager.acquireLock(data.sandboxId, async () => { + try { + terminals[id] = await containers[data.sandboxId].terminal.start({ + onData: (data: string) => { + io.emit("terminalResponse", { id, data }); + }, + size: { cols: 80, rows: 20 }, + onExit: () => console.log("Terminal exited", id), + }); + await terminals[id].sendData( + `cd "${path.join(dirName, "projects", data.sandboxId)}"\r` + ); + await terminals[id].sendData("export PS1='user> '\rclear\r"); + console.log("Created terminal", id); + } catch (e: any) { + console.error(`Error creating terminal ${id}:`, e); + io.emit("error", `Error: terminal creation. ${e.message ?? e}`); + } + }); + + callback(); + } catch (e: any) { + console.error(`Error creating terminal ${id}:`, e); + io.emit("error", `Error: terminal creation. ${e.message ?? e}`); + } + }); + + socket.on( + "resizeTerminal", + (dimensions: { cols: number; rows: number }) => { + try { + Object.values(terminals).forEach((t) => { + t.resize(dimensions); + }); + } catch (e: any) { + console.error("Error resizing terminal:", e); + io.emit("error", `Error: terminal resizing. ${e.message ?? e}`); + } + } + ); + + socket.on("terminalData", (id: string, data: string) => { + try { + if (!terminals[id]) { + return; + } + + terminals[id].sendData(data); + } catch (e: any) { + console.error("Error writing to terminal:", e); + io.emit("error", `Error: writing to terminal. ${e.message ?? e}`); + } + }); + + socket.on("closeTerminal", async (id: string, callback) => { + try { + if (!terminals[id]) { + return; + } + + await terminals[id].kill(); + delete terminals[id]; + + callback(); + } catch (e: any) { + console.error("Error closing terminal:", e); + io.emit("error", `Error: closing terminal. ${e.message ?? e}`); + } + }); + + socket.on( + "generateCode", + async ( + fileName: string, + code: string, + line: number, + instructions: string, + callback + ) => { + try { + const fetchPromise = fetch( + `${process.env.DATABASE_WORKER_URL}/api/sandbox/generate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ + userId: data.userId, + }), + } + ); + + // Generate code from cloudflare workers AI + const generateCodePromise = fetch( + `${process.env.AI_WORKER_URL}/api?fileName=${fileName}&code=${code}&line=${line}&instructions=${instructions}`, + { + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.CF_AI_KEY}`, + }, + } + ); + + const [fetchResponse, generateCodeResponse] = await Promise.all([ + fetchPromise, + generateCodePromise, + ]); + + const json = await generateCodeResponse.json(); + + callback({ response: json.response, success: true }); + } catch (e: any) { + console.error("Error generating code:", e); + io.emit("error", `Error: code generation. ${e.message ?? e}`); + } + } + ); + + socket.on("disconnect", async () => { + try { + if (data.isOwner) { + connections[data.sandboxId]--; + } + + if (data.isOwner && connections[data.sandboxId] <= 0) { + await Promise.all( + Object.entries(terminals).map(async ([key, terminal]) => { + await terminal.kill(); + delete terminals[key]; + }) + ); + + await lockManager.acquireLock(data.sandboxId, async () => { + try { + if (containers[data.sandboxId]) { + await containers[data.sandboxId].close(); + delete containers[data.sandboxId]; + console.log("Closed container", data.sandboxId); + } + } catch (error) { + console.error("Error closing container ", data.sandboxId, error); + } + }); + + socket.broadcast.emit( + "disableAccess", + "The sandbox owner has disconnected." + ); + } + + // const sockets = await io.fetchSockets(); + // if (inactivityTimeout) { + // clearTimeout(inactivityTimeout); + // } + // if (sockets.length === 0) { + // console.log("STARTING TIMER"); + // inactivityTimeout = setTimeout(() => { + // io.fetchSockets().then(async (sockets) => { + // if (sockets.length === 0) { + // console.log("Server stopped", res); + // } + // }); + // }, 20000); + // } else { + // console.log("number of sockets", sockets.length); + // } + } catch (e: any) { + console.log("Error disconnecting:", e); + io.emit("error", `Error: disconnecting. ${e.message ?? e}`); + } + }); + } catch (e: any) { + console.error("Error connecting:", e); + io.emit("error", `Error: connection. ${e.message ?? e}`); + } }); httpServer.listen(port, () => { diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index d88c2e8..05699f1 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -391,7 +391,7 @@ export default function CodeEditor({ setFiles(files) } - const onRateLimit = (message: string) => { + const onError = (message: string) => { toast.error(message) } @@ -413,7 +413,7 @@ export default function CodeEditor({ socketRef.current?.on("connect", onConnect) socketRef.current?.on("disconnect", onDisconnect) socketRef.current?.on("loaded", onLoadedEvent) - socketRef.current?.on("rateLimit", onRateLimit) + socketRef.current?.on("error", onError) socketRef.current?.on("terminalResponse", onTerminalResponse) socketRef.current?.on("disableAccess", onDisableAccess) socketRef.current?.on("previewURL", setPreviewURL) @@ -422,7 +422,7 @@ export default function CodeEditor({ socketRef.current?.off("connect", onConnect) socketRef.current?.off("disconnect", onDisconnect) socketRef.current?.off("loaded", onLoadedEvent) - socketRef.current?.off("rateLimit", onRateLimit) + socketRef.current?.off("error", onError) socketRef.current?.off("terminalResponse", onTerminalResponse) socketRef.current?.off("disableAccess", onDisableAccess) socketRef.current?.off("previewURL", setPreviewURL) From 6f6926a6213cf4a28a865988675ba1c445f19cc6 Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Mon, 15 Jul 2024 14:56:37 -0400 Subject: [PATCH 13/29] fix: store rooms in map --- frontend/components/editor/index.tsx | 96 ++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 05699f1..45d4962 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -104,6 +104,16 @@ export default function CodeEditor({ const room = useRoom() const [provider, setProvider] = useState() const userInfo = useSelf((me) => me.info) + + // Liveblocks providers map to prevent reinitializing providers + type ProviderData = { + provider: LiveblocksProvider; + yDoc: Y.Doc; + yText: Y.Text; + binding?: MonacoBinding; + onSync: (isSynced: boolean) => void; + }; + const providersMap = useRef(new Map()); // Refs for libraries / features const editorContainerRef = useRef(null) @@ -332,43 +342,77 @@ export default function CodeEditor({ if (!editorRef || !tab || !model) return - const yDoc = new Y.Doc() - const yText = yDoc.getText(tab.id) - const yProvider: any = new LiveblocksProvider(room, yDoc) + let providerData: ProviderData; + + // When a file is opened for the first time, create a new provider and store in providersMap. + if (!providersMap.current.has(tab.id)) { + const yDoc = new Y.Doc(); + const yText = yDoc.getText(tab.id); + const yProvider = new LiveblocksProvider(room, yDoc); - const onSync = (isSynced: boolean) => { - if (isSynced) { - const text = yText.toString() - if (text === "") { - if (activeFileContent) { - yText.insert(0, activeFileContent) - } else { - setTimeout(() => { - yText.insert(0, editorRef.getValue()) - }, 0) + // Inserts the file content into the editor once when the tab is changed. + const onSync = (isSynced: boolean) => { + if (isSynced) { + const text = yText.toString() + if (text === "") { + if (activeFileContent) { + yText.insert(0, activeFileContent) + } else { + setTimeout(() => { + yText.insert(0, editorRef.getValue()) + }, 0) + } } } } + + yProvider.on("sync", onSync) + + // Save the provider to the map. + providerData = { provider: yProvider, yDoc, yText, onSync }; + providersMap.current.set(tab.id, providerData); + + } else { + // When a tab is opened that has been open before, reuse the existing provider. + providerData = providersMap.current.get(tab.id)!; } - yProvider.on("sync", onSync) - - setProvider(yProvider) - const binding = new MonacoBinding( - yText, + providerData.yText, model, new Set([editorRef]), - yProvider.awareness as Awareness - ) + providerData.provider.awareness as unknown as Awareness + ); + + providerData.binding = binding; + + setProvider(providerData.provider); return () => { - yDoc.destroy() - yProvider.destroy() - binding.destroy() - yProvider.off("sync", onSync) - } - }, [editorRef, room, activeFileContent]) + // Cleanup logic + if (binding) { + binding.destroy(); + } + if (providerData.binding) { + providerData.binding = undefined; + } + }; + }, [editorRef, room, activeFileContent, activeFileId, tabs]); + + // Added this effect to clean up when the component unmounts + useEffect(() => { + return () => { + // Clean up all providers when the component unmounts + providersMap.current.forEach((data) => { + if (data.binding) { + data.binding.destroy(); + } + data.provider.disconnect(); + data.yDoc.destroy(); + }); + providersMap.current.clear(); + }; + }, []); // Connection/disconnection effect useEffect(() => { From 5bf264b807a1ec72b5c95dd2bbc670dcb175983b Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Mon, 15 Jul 2024 15:32:40 -0400 Subject: [PATCH 14/29] fix: remove extra state variables from useEffect --- frontend/components/editor/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 45d4962..46fb9ab 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -397,7 +397,7 @@ export default function CodeEditor({ providerData.binding = undefined; } }; - }, [editorRef, room, activeFileContent, activeFileId, tabs]); + }, [editorRef, room, activeFileContent]); // Added this effect to clean up when the component unmounts useEffect(() => { From 7dd67f72d8747d8b225ff6c9c3a4e48851a55ef8 Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Mon, 15 Jul 2024 16:12:08 -0400 Subject: [PATCH 15/29] fix: remove editorRef from useEffect --- frontend/components/editor/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 46fb9ab..495951e 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -397,7 +397,7 @@ export default function CodeEditor({ providerData.binding = undefined; } }; - }, [editorRef, room, activeFileContent]); + }, [room, activeFileContent]); // Added this effect to clean up when the component unmounts useEffect(() => { From 478a332a2e6f0067d6eafa21a3fec4070fc8bf3a Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Wed, 17 Jul 2024 11:30:45 -0400 Subject: [PATCH 16/29] feat: added deploy button --- frontend/components/editor/navbar/deploy.tsx | 28 ++++++++++++++++++++ frontend/components/editor/navbar/index.tsx | 12 ++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 frontend/components/editor/navbar/deploy.tsx diff --git a/frontend/components/editor/navbar/deploy.tsx b/frontend/components/editor/navbar/deploy.tsx new file mode 100644 index 0000000..b40ec20 --- /dev/null +++ b/frontend/components/editor/navbar/deploy.tsx @@ -0,0 +1,28 @@ +"use client"; + +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Play, Pause } from "lucide-react"; + +export default function DeployButtonModal() { + const [isDeploying, setIsDeploying] = useState(false); + + const handleDeploy = () => { + if (isDeploying) { + console.log("Stopping deployment..."); + + } else { + console.log("Starting deployment..."); + } + setIsDeploying(!isDeploying); + }; + + return ( + <> + + + ); +} diff --git a/frontend/components/editor/navbar/index.tsx b/frontend/components/editor/navbar/index.tsx index 413200b..9c4a123 100644 --- a/frontend/components/editor/navbar/index.tsx +++ b/frontend/components/editor/navbar/index.tsx @@ -11,6 +11,7 @@ import { useState } from "react"; import EditSandboxModal from "./edit"; import ShareSandboxModal from "./share"; import { Avatars } from "../live/avatars"; +import DeployButtonModal from "./deploy"; export default function Navbar({ userData, @@ -66,10 +67,13 @@ export default function Navbar({ {isOwner ? ( - + <> + + + ) : null} From dead84ac4da4460b7174611a4d6fed121382a685 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Thu, 4 Jul 2024 20:50:35 -0400 Subject: [PATCH 17/29] fix: make server url an environment variable --- frontend/.env.example | 2 +- frontend/components/editor/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/.env.example b/frontend/.env.example index e6a58a9..e81c748 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -3,7 +3,7 @@ CLERK_SECRET_KEY= NEXT_PUBLIC_LIVEBLOCKS_PUBLIC_KEY= LIVEBLOCKS_SECRET_KEY= -NEXT_PUBLIC_SERVER_PORT=4000 +NEXT_PUBLIC_SERVER_URL=http://localhost:4000 NEXT_PUBLIC_APP_URL=http://localhost:3000 # Set WORKER_URLs after deploying the workers. diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 30c8d60..8ae8373 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -44,7 +44,7 @@ export default function CodeEditor({ // Initialize socket connection if it doesn't exist if (!socketRef.current) { socketRef.current = io( - `${window.location.protocol}//${window.location.hostname}:${process.env.NEXT_PUBLIC_SERVER_PORT}?userId=${userData.id}&sandboxId=${sandboxData.id}`, + `${process.env.NEXT_PUBLIC_SERVER_URL}?userId=${userData.id}&sandboxId=${sandboxData.id}`, { timeout: 2000, } From 769f52816f35239684b1c1acf2d4473dbf269174 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sun, 21 Jul 2024 14:58:38 -0400 Subject: [PATCH 18/29] Add Dokku connection and test client. --- backend/server/src/index.ts | 48 ++++++ tests/index.ts | 39 +++++ tests/package-lock.json | 310 ++++++++++++++++++++++++++++++++++++ tests/package.json | 21 +++ tests/tsconfig.json | 13 ++ 5 files changed, 431 insertions(+) create mode 100644 tests/index.ts create mode 100644 tests/package-lock.json create mode 100644 tests/package.json create mode 100644 tests/tsconfig.json diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 733d366..6c74f26 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -5,6 +5,7 @@ import express, { Express } from "express"; import dotenv from "dotenv"; import { createServer } from "http"; import { Server } from "socket.io"; +import net from "net"; import { z } from "zod"; import { User } from "./types"; @@ -112,6 +113,21 @@ io.use(async (socket, next) => { const lockManager = new LockManager(); +const client = net.createConnection( + { path: "/var/run/remote-dokku.sock" }, + () => { + console.log("Connected to Dokku server"); + } +); + +client.on("end", () => { + console.log("Disconnected from Dokku server"); +}); + +client.on("error", (err) => { + console.error(`Dokku Client error: ${err}`); +}); + io.on("connection", async (socket) => { try { if (inactivityTimeout) clearTimeout(inactivityTimeout); @@ -254,6 +270,38 @@ io.on("connection", async (socket) => { } ); + interface DokkuResponse { + ok: boolean; + output: string; + } + + interface CallbackResponse { + success: boolean; + apps?: string[]; + message?: string; + } + + socket.on( + "list", + async (callback: (response: CallbackResponse) => void) => { + console.log("Retrieving apps list..."); + client.on("data", (data) => { + const response = data.toString(); + const parsedData: DokkuResponse = JSON.parse(response); + if (parsedData.ok) { + const appsList = parsedData.output.split("\n").slice(1); // Split by newline and ignore the first line (header) + callback({ success: true, apps: appsList }); + } else { + callback({ + success: false, + message: "Failed to retrieve apps list", + }); + } + }); + client.write("apps:list\n"); + } + ); + socket.on("createFile", async (name: string, callback) => { try { const size: number = await getProjectSize(data.sandboxId); diff --git a/tests/index.ts b/tests/index.ts new file mode 100644 index 0000000..e2fc1f4 --- /dev/null +++ b/tests/index.ts @@ -0,0 +1,39 @@ +// Import necessary modules +import { io, Socket } from "socket.io-client"; +import dotenv from "dotenv"; + +dotenv.config(); + +interface CallbackResponse { + success: boolean; + apps?: string[]; + message?: string; +} + +let socketRef: Socket = io( + `http://localhost:4000?userId=user_2hFB6KcK6bb3Gx9241UXsxFq4kO&sandboxId=aabuk4vneecj2csni24kpabv`, + { + timeout: 2000, + } +); + +socketRef.on("connect", async () => { + console.log("Connected to the server"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + + socketRef.emit("list", (response: CallbackResponse) => { + if (response.success) { + console.log("List of apps:", response.apps); + } else { + console.log("Error:", response.message); + } + }); +}); + +socketRef.on("disconnect", () => { + console.log("Disconnected from the server"); +}); + +socketRef.on("connect_error", (error: Error) => { + console.error("Connection error:", error); +}); diff --git a/tests/package-lock.json b/tests/package-lock.json new file mode 100644 index 0000000..f394c33 --- /dev/null +++ b/tests/package-lock.json @@ -0,0 +1,310 @@ +{ + "name": "socket-io-test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "socket-io-test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "dotenv": "^16.4.5", + "socket.io-client": "^4.7.5", + "ts-node": "^10.9.2" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@socket.io/component-emitter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", + "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==" + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==" + }, + "node_modules/@types/node": { + "version": "20.14.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", + "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", + "peer": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/acorn": { + "version": "8.12.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", + "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.3", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.3.tgz", + "integrity": "sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==" + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" + }, + "node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/engine.io-client": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.4.tgz", + "integrity": "sha512-GeZeeRjpD2qf49cZQ0Wvh/8NJNfeXkXXcoGh+F77oEAgo9gUHwT1fCRxSNU+YEEaysOJTnsFHmM5oAcPy4ntvQ==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1", + "engine.io-parser": "~5.2.1", + "ws": "~8.17.1", + "xmlhttprequest-ssl": "~2.0.0" + } + }, + "node_modules/engine.io-parser": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", + "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/socket.io-client": { + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.7.5.tgz", + "integrity": "sha512-sJ/tqHOCe7Z50JCBCXrsY3I2k03iOiUe+tj1OmKeD2lXPiGH/RUCdTZFoqVyN7l1MnpIzPrGtLcijffmeouNlQ==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.2", + "engine.io-client": "~6.5.2", + "socket.io-parser": "~4.2.4" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/socket.io-parser": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz", + "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/typescript": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "peer": true + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==" + }, + "node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xmlhttprequest-ssl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz", + "integrity": "sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "engines": { + "node": ">=6" + } + } + } +} diff --git a/tests/package.json b/tests/package.json new file mode 100644 index 0000000..42eef8c --- /dev/null +++ b/tests/package.json @@ -0,0 +1,21 @@ +{ + "name": "socket-io-test", + "version": "1.0.0", + "description": "A test script for socket.io-client using ES6 modules and TypeScript", + "main": "dist/index.js", + "type": "module", + "scripts": { + "build": "tsc", + "start": "npm run build && node dist/index.js" + }, + "author": "Your Name", + "license": "ISC", + "dependencies": { + "dotenv": "^16.4.5", + "socket.io-client": "^4.7.5" + }, + "devDependencies": { + "typescript": "^5.0.0", + "ts-node": "^10.9.2" + } +} \ No newline at end of file diff --git a/tests/tsconfig.json b/tests/tsconfig.json new file mode 100644 index 0000000..694c381 --- /dev/null +++ b/tests/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "node", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist" + }, + "include": ["index.ts"] +} \ No newline at end of file From de4923ec1eec80728032c72977f56426da43ff92 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sun, 21 Jul 2024 14:18:14 -0400 Subject: [PATCH 19/29] Connect to remote Dokku server using SSH. --- backend/server/package-lock.json | 87 ++++++++++++++++++++++++++ backend/server/package.json | 2 + backend/server/src/DokkuClient.ts | 37 +++++++++++ backend/server/src/SSHSocketClient.ts | 90 +++++++++++++++++++++++++++ backend/server/src/index.ts | 52 +++++++--------- 5 files changed, 237 insertions(+), 31 deletions(-) create mode 100644 backend/server/src/DokkuClient.ts create mode 100644 backend/server/src/SSHSocketClient.ts diff --git a/backend/server/package-lock.json b/backend/server/package-lock.json index 8c7b2a0..d167660 100644 --- a/backend/server/package-lock.json +++ b/backend/server/package-lock.json @@ -16,12 +16,14 @@ "express": "^4.19.2", "rate-limiter-flexible": "^5.0.3", "socket.io": "^4.7.5", + "ssh2": "^1.15.0", "zod": "^3.22.4" }, "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^4.17.21", "@types/node": "^20.12.7", + "@types/ssh2": "^1.15.0", "nodemon": "^3.1.0", "ts-node": "^10.9.2", "typescript": "^5.4.5" @@ -213,6 +215,24 @@ "@types/send": "*" } }, + "node_modules/@types/ssh2": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.0.tgz", + "integrity": "sha512-YcT8jP5F8NzWeevWvcyrrLB3zcneVjzYY9ZDSMAMboI+2zR1qYWFhwsyOFVzT7Jorn67vqxC0FRiw8YyG9P1ww==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.41.tgz", + "integrity": "sha512-LX84pRJ+evD2e2nrgYCHObGWkiQJ1mL+meAgbvnwk/US6vmMY7S2ygBTGV2Jw91s9vUsLSXeDEkUHZIJGLrhsg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -298,6 +318,14 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -312,6 +340,14 @@ "node": "^4.5.0 || >= 5.9" } }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -382,6 +418,15 @@ "node": ">=6.14.2" } }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -593,6 +638,20 @@ "node": ">= 0.10" } }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -1247,6 +1306,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/nan": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", + "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", + "optional": true + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -1748,6 +1813,23 @@ "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz", "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==" }, + "node_modules/ssh2": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz", + "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==", + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.9", + "nan": "^2.18.0" + } + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1880,6 +1962,11 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", diff --git a/backend/server/package.json b/backend/server/package.json index 3b89d67..dc0f09c 100644 --- a/backend/server/package.json +++ b/backend/server/package.json @@ -18,12 +18,14 @@ "express": "^4.19.2", "rate-limiter-flexible": "^5.0.3", "socket.io": "^4.7.5", + "ssh2": "^1.15.0", "zod": "^3.22.4" }, "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^4.17.21", "@types/node": "^20.12.7", + "@types/ssh2": "^1.15.0", "nodemon": "^3.1.0", "ts-node": "^10.9.2", "typescript": "^5.4.5" diff --git a/backend/server/src/DokkuClient.ts b/backend/server/src/DokkuClient.ts new file mode 100644 index 0000000..fd0adcd --- /dev/null +++ b/backend/server/src/DokkuClient.ts @@ -0,0 +1,37 @@ +import { SSHSocketClient, SSHConfig } from "./SSHSocketClient" + +export interface DokkuResponse { + ok: boolean; + output: string; +} + +export class DokkuClient extends SSHSocketClient { + + constructor(config: SSHConfig) { + super( + config, + "/var/run/dokku-daemon/dokku-daemon.sock" + ) + } + + async sendCommand(command: string): Promise { + try { + const response = await this.sendData(command); + + if (typeof response !== "string") { + throw new Error("Received data is not a string"); + } + + return JSON.parse(response); + } catch (error: any) { + throw new Error(`Failed to send command: ${error.message}`); + } + } + + async listApps(): Promise { + const response = await this.sendCommand("apps:list"); + return response.output.split("\n").slice(1); // Split by newline and ignore the first line (header) + } +} + +export { SSHConfig }; \ No newline at end of file diff --git a/backend/server/src/SSHSocketClient.ts b/backend/server/src/SSHSocketClient.ts new file mode 100644 index 0000000..e0dc043 --- /dev/null +++ b/backend/server/src/SSHSocketClient.ts @@ -0,0 +1,90 @@ +import { Client } from "ssh2"; + +export interface SSHConfig { + host: string; + port?: number; + username: string; + privateKey: Buffer; +} + +export class SSHSocketClient { + private conn: Client; + private config: SSHConfig; + private socketPath: string; + private isConnected: boolean = false; + + constructor(config: SSHConfig, socketPath: string) { + this.conn = new Client(); + this.config = { ...config, port: 22}; + this.socketPath = socketPath; + + this.setupTerminationHandlers(); + } + + private setupTerminationHandlers() { + process.on("SIGINT", this.closeConnection.bind(this)); + process.on("SIGTERM", this.closeConnection.bind(this)); + } + + private closeConnection() { + console.log("Closing SSH connection..."); + this.conn.end(); + this.isConnected = false; + process.exit(0); + } + + connect(): Promise { + return new Promise((resolve, reject) => { + this.conn + .on("ready", () => { + console.log("SSH connection established"); + this.isConnected = true; + resolve(); + }) + .on("error", (err) => { + console.error("SSH connection error:", err); + this.isConnected = false; + reject(err); + }) + .on("close", () => { + console.log("SSH connection closed"); + this.isConnected = false; + }) + .connect(this.config); + }); + } + + sendData(data: string): Promise { + return new Promise((resolve, reject) => { + if (!this.isConnected) { + reject(new Error("SSH connection is not established")); + return; + } + + this.conn.exec( + `echo "${data}" | nc -U ${this.socketPath}`, + (err, stream) => { + if (err) { + reject(err); + return; + } + + stream + .on("close", (code: number, signal: string) => { + reject( + new Error( + `Stream closed with code ${code} and signal ${signal}` + ) + ); + }) + .on("data", (data: Buffer) => { + resolve(data.toString()); + }) + .stderr.on("data", (data: Buffer) => { + reject(new Error(data.toString())); + }); + } + ); + }); + } + } \ No newline at end of file diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 6c74f26..187db67 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -5,7 +5,8 @@ import express, { Express } from "express"; import dotenv from "dotenv"; import { createServer } from "http"; import { Server } from "socket.io"; -import net from "net"; +import { DokkuClient, SSHConfig } from "./DokkuClient"; +import fs from "fs"; import { z } from "zod"; import { User } from "./types"; @@ -113,20 +114,17 @@ io.use(async (socket, next) => { const lockManager = new LockManager(); -const client = net.createConnection( - { path: "/var/run/remote-dokku.sock" }, - () => { - console.log("Connected to Dokku server"); - } -); +if (!process.env.DOKKU_HOST) throw new Error('Environment variable DOKKU_HOST is not defined'); +if (!process.env.DOKKU_USERNAME) throw new Error('Environment variable DOKKU_USERNAME is not defined'); +if (!process.env.DOKKU_KEY) throw new Error('Environment variable DOKKU_KEY is not defined'); -client.on("end", () => { - console.log("Disconnected from Dokku server"); +const client = new DokkuClient({ + host: process.env.DOKKU_HOST, + username: process.env.DOKKU_USERNAME, + privateKey: fs.readFileSync(process.env.DOKKU_KEY), }); -client.on("error", (err) => { - console.error(`Dokku Client error: ${err}`); -}); +client.connect(); io.on("connection", async (socket) => { try { @@ -270,11 +268,6 @@ io.on("connection", async (socket) => { } ); - interface DokkuResponse { - ok: boolean; - output: string; - } - interface CallbackResponse { success: boolean; apps?: string[]; @@ -285,20 +278,17 @@ io.on("connection", async (socket) => { "list", async (callback: (response: CallbackResponse) => void) => { console.log("Retrieving apps list..."); - client.on("data", (data) => { - const response = data.toString(); - const parsedData: DokkuResponse = JSON.parse(response); - if (parsedData.ok) { - const appsList = parsedData.output.split("\n").slice(1); // Split by newline and ignore the first line (header) - callback({ success: true, apps: appsList }); - } else { - callback({ - success: false, - message: "Failed to retrieve apps list", - }); - } - }); - client.write("apps:list\n"); + try { + callback({ + success: true, + apps: await client.listApps() + }); + } catch (error) { + callback({ + success: false, + message: "Failed to retrieve apps list", + }); + } } ); From deb32352fb15648d0be7225ea910a508273c9a81 Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Tue, 23 Jul 2024 17:30:35 -0400 Subject: [PATCH 20/29] feat: add run button --- frontend/app/layout.tsx | 8 +- frontend/components/editor/index.tsx | 454 +++++++++--------- frontend/components/editor/navbar/index.tsx | 19 +- frontend/components/editor/navbar/run.tsx | 59 +++ frontend/components/editor/preview/index.tsx | 25 +- .../components/editor/terminals/index.tsx | 100 ++-- frontend/context/PreviewContext.tsx | 34 ++ frontend/context/TerminalContext.tsx | 117 +++++ 8 files changed, 518 insertions(+), 298 deletions(-) create mode 100644 frontend/components/editor/navbar/run.tsx create mode 100644 frontend/context/PreviewContext.tsx create mode 100644 frontend/context/TerminalContext.tsx diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsx index 50ee950..79f8b5d 100644 --- a/frontend/app/layout.tsx +++ b/frontend/app/layout.tsx @@ -6,6 +6,8 @@ import { ThemeProvider } from "@/components/layout/themeProvider" import { ClerkProvider } from "@clerk/nextjs" import { Toaster } from "@/components/ui/sonner" import { Analytics } from "@vercel/analytics/react" +import { TerminalProvider } from '@/context/TerminalContext'; +import { PreviewProvider } from "@/context/PreviewContext" export const metadata: Metadata = { title: "Sandbox", @@ -27,7 +29,11 @@ export default function RootLayout({ forcedTheme="dark" disableTransitionOnChange > + + {children} + + @@ -35,4 +41,4 @@ export default function RootLayout({ ) -} +} \ No newline at end of file diff --git a/frontend/components/editor/index.tsx b/frontend/components/editor/index.tsx index 495951e..a00fcd8 100644 --- a/frontend/components/editor/index.tsx +++ b/frontend/components/editor/index.tsx @@ -31,6 +31,8 @@ import Loading from "./loading" import PreviewWindow from "./preview" import Terminals from "./terminals" import { ImperativePanelHandle } from "react-resizable-panels" +import { PreviewProvider, usePreview } from '@/context/PreviewContext'; +import { useTerminal } from '@/context/TerminalContext'; export default function CodeEditor({ userData, @@ -50,8 +52,17 @@ export default function CodeEditor({ { timeout: 2000, } - );} + ); + } + //Terminalcontext functionsand effects + const { setUserAndSandboxId } = useTerminal(); + + useEffect(() => { + setUserAndSandboxId(userData.id, sandboxData.id); + }, [userData.id, sandboxData.id, setUserAndSandboxId]); + + //Preview Button state const [isPreviewCollapsed, setIsPreviewCollapsed] = useState(true) const [disableAccess, setDisableAccess] = useState({ isDisabled: false, @@ -104,7 +115,7 @@ export default function CodeEditor({ const room = useRoom() const [provider, setProvider] = useState() const userInfo = useSelf((me) => me.info) - + // Liveblocks providers map to prevent reinitializing providers type ProviderData = { provider: LiveblocksProvider; @@ -317,7 +328,7 @@ export default function CodeEditor({ console.log(`Saving file...${activeFileId}`); console.log(`Saving file...${value}`); socketRef.current?.emit("saveFile", activeFileId, value); - }, Number(process.env.FILE_SAVE_DEBOUNCE_DELAY)||1000), + }, Number(process.env.FILE_SAVE_DEBOUNCE_DELAY) || 1000), [socketRef] ); @@ -343,7 +354,7 @@ export default function CodeEditor({ if (!editorRef || !tab || !model) return let providerData: ProviderData; - + // When a file is opened for the first time, create a new provider and store in providersMap. if (!providersMap.current.has(tab.id)) { const yDoc = new Y.Doc(); @@ -385,7 +396,7 @@ export default function CodeEditor({ ); providerData.binding = binding; - + setProvider(providerData.provider); return () => { @@ -399,25 +410,25 @@ export default function CodeEditor({ }; }, [room, activeFileContent]); - // Added this effect to clean up when the component unmounts - useEffect(() => { - return () => { - // Clean up all providers when the component unmounts - providersMap.current.forEach((data) => { - if (data.binding) { - data.binding.destroy(); - } - data.provider.disconnect(); - data.yDoc.destroy(); - }); - providersMap.current.clear(); - }; - }, []); + // Added this effect to clean up when the component unmounts + useEffect(() => { + return () => { + // Clean up all providers when the component unmounts + providersMap.current.forEach((data) => { + if (data.binding) { + data.binding.destroy(); + } + data.provider.disconnect(); + data.yDoc.destroy(); + }); + providersMap.current.clear(); + }; + }, []); // Connection/disconnection effect useEffect(() => { socketRef.current?.connect() - + return () => { socketRef.current?.disconnect() } @@ -425,7 +436,7 @@ export default function CodeEditor({ // Socket event listener effect useEffect(() => { - const onConnect = () => {} + const onConnect = () => { } const onDisconnect = () => { setTerminals([]) @@ -530,8 +541,8 @@ export default function CodeEditor({ ? numTabs === 1 ? null : index < numTabs - 1 - ? tabs[index + 1].id - : tabs[index - 1].id + ? tabs[index + 1].id + : tabs[index - 1].id : activeFileId setTabs((prev) => prev.filter((t) => t.id !== id)) @@ -624,7 +635,7 @@ export default function CodeEditor({ {}} + setOpen={() => { }} /> @@ -633,216 +644,211 @@ export default function CodeEditor({ return ( <> {/* Copilot DOM elements */} -
-
- {generate.show && ai ? ( - t.id === activeFileId)?.name ?? "", - code: editorRef?.getValue() ?? "", - line: generate.line, - }} - editor={{ - language: editorLanguage, - }} - onExpand={() => { - editorRef?.changeViewZones(function (changeAccessor) { - changeAccessor.removeZone(generate.id) + +
+
+ {generate.show && ai ? ( + t.id === activeFileId)?.name ?? "", + code: editorRef?.getValue() ?? "", + line: generate.line, + }} + editor={{ + language: editorLanguage, + }} + onExpand={() => { + editorRef?.changeViewZones(function (changeAccessor) { + changeAccessor.removeZone(generate.id) - if (!generateRef.current) return - const id = changeAccessor.addZone({ - afterLineNumber: cursorLine, - heightInLines: 12, - domNode: generateRef.current, + if (!generateRef.current) return + const id = changeAccessor.addZone({ + afterLineNumber: cursorLine, + heightInLines: 12, + domNode: generateRef.current, + }) + setGenerate((prev) => { + return { ...prev, id } + }) }) + }} + onAccept={(code: string) => { + const line = generate.line setGenerate((prev) => { - return { ...prev, id } + return { + ...prev, + show: !prev.show, + } }) - }) - }} - onAccept={(code: string) => { - const line = generate.line - setGenerate((prev) => { - return { - ...prev, - show: !prev.show, - } - }) - const file = editorRef?.getValue() + const file = editorRef?.getValue() - const lines = file?.split("\n") || [] - lines.splice(line - 1, 0, code) - const updatedFile = lines.join("\n") - editorRef?.setValue(updatedFile) - }} - onClose={() => { - setGenerate((prev) => { - return { - ...prev, - show: !prev.show, - } - }) - }} - /> - ) : null} -
+ const lines = file?.split("\n") || [] + lines.splice(line - 1, 0, code) + const updatedFile = lines.join("\n") + editorRef?.setValue(updatedFile) + }} + onClose={() => { + setGenerate((prev) => { + return { + ...prev, + show: !prev.show, + } + }) + }} + /> + ) : null} +
- {/* Main editor components */} - addNew(name, type, setFiles, sandboxData)} - deletingFolderId={deletingFolderId} - // AI Copilot Toggle - ai={ai} - setAi={setAi} - /> + {/* Main editor components */} + addNew(name, type, setFiles, sandboxData)} + deletingFolderId={deletingFolderId} + // AI Copilot Toggle + ai={ai} + setAi={setAi} + /> - {/* Shadcn resizeable panels: https://ui.shadcn.com/docs/components/resizable */} - - -
- {/* File tabs */} - {tabs.map((tab) => ( - { - selectFile(tab) - }} - onClose={() => closeTab(tab.id)} - > - {tab.name} - - ))} -
- {/* Monaco editor */} -
+ - {!activeFileId ? ( - <> -
- - No file selected. -
- - ) : // Note clerk.loaded is required here due to a bug: https://github.com/clerk/javascript/issues/1643 - clerk.loaded ? ( - <> - {provider && userInfo ? ( - - ) : null} - { - if (value === activeFileContent) { - setTabs((prev) => - prev.map((tab) => - tab.id === activeFileId - ? { ...tab, saved: true } - : tab - ) - ) - } else { - setTabs((prev) => - prev.map((tab) => - tab.id === activeFileId - ? { ...tab, saved: false } - : tab - ) - ) - } +
+ {/* File tabs */} + {tabs.map((tab) => ( + { + selectFile(tab) }} - options={{ - tabSize: 2, - minimap: { - enabled: false, - }, - padding: { - bottom: 4, - top: 4, - }, - scrollBeyondLastLine: false, - fixedOverflowWidgets: true, - fontFamily: "var(--font-geist-mono)", - }} - theme="vs-dark" - value={activeFileContent} - /> - - ) : ( -
- - Waiting for Clerk to load... -
- )} -
- - - - - setIsPreviewCollapsed(true)} - onExpand={() => setIsPreviewCollapsed(false)} + onClose={() => closeTab(tab.id)} + > + {tab.name} + + ))} +
+ {/* Monaco editor */} +
- { - previewPanelRef.current?.expand() - setIsPreviewCollapsed(false) - }} - src={previewURL} - /> - - - - {isOwner ? ( - - ) : ( -
- - No terminal access. -
- )} - - - - + {!activeFileId ? ( + <> +
+ + No file selected. +
+ + ) : // Note clerk.loaded is required here due to a bug: https://github.com/clerk/javascript/issues/1643 + clerk.loaded ? ( + <> + {provider && userInfo ? ( + + ) : null} + { + if (value === activeFileContent) { + setTabs((prev) => + prev.map((tab) => + tab.id === activeFileId + ? { ...tab, saved: true } + : tab + ) + ) + } else { + setTabs((prev) => + prev.map((tab) => + tab.id === activeFileId + ? { ...tab, saved: false } + : tab + ) + ) + } + }} + options={{ + tabSize: 2, + minimap: { + enabled: false, + }, + padding: { + bottom: 4, + top: 4, + }, + scrollBeyondLastLine: false, + fixedOverflowWidgets: true, + fontFamily: "var(--font-geist-mono)", + }} + theme="vs-dark" + value={activeFileContent} + /> + + ) : ( +
+ + Waiting for Clerk to load... +
+ )} +
+ + + + + setIsPreviewCollapsed(true)} + onExpand={() => setIsPreviewCollapsed(false)} + > + { + usePreview().previewPanelRef.current?.expand() + setIsPreviewCollapsed(false) + } } collapsed={isPreviewCollapsed} src={previewURL}/> + + + + {isOwner ? ( + + ) : ( +
+ + No terminal access. +
+ )} + + + + + ) } diff --git a/frontend/components/editor/navbar/index.tsx b/frontend/components/editor/navbar/index.tsx index 413200b..652fa03 100644 --- a/frontend/components/editor/navbar/index.tsx +++ b/frontend/components/editor/navbar/index.tsx @@ -11,6 +11,8 @@ import { useState } from "react"; import EditSandboxModal from "./edit"; import ShareSandboxModal from "./share"; import { Avatars } from "../live/avatars"; +import RunButtonModal from "./run"; +import DeployButtonModal from "./deploy"; export default function Navbar({ userData, @@ -19,15 +21,13 @@ export default function Navbar({ }: { userData: User; sandboxData: Sandbox; - shared: { - id: string; - name: string; - }[]; + shared: { id: string; name: string }[]; }) { const [isEditOpen, setIsEditOpen] = useState(false); const [isShareOpen, setIsShareOpen] = useState(false); + const [isRunning, setIsRunning] = useState(false); - const isOwner = sandboxData.userId === userData.id; + const isOwner = sandboxData.userId === userData.id;; return ( <> @@ -62,18 +62,25 @@ export default function Navbar({ ) : null}
+
{isOwner ? ( + <> + + ) : null}
); -} +} \ No newline at end of file diff --git a/frontend/components/editor/navbar/run.tsx b/frontend/components/editor/navbar/run.tsx new file mode 100644 index 0000000..688886f --- /dev/null +++ b/frontend/components/editor/navbar/run.tsx @@ -0,0 +1,59 @@ +"use client"; + +import { Play, StopCircle } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { useTerminal } from "@/context/TerminalContext"; +import { usePreview } from "@/context/PreviewContext"; +import { toast } from "sonner"; + +export default function RunButtonModal({ + isRunning, + setIsRunning, +}: { + isRunning: boolean; + setIsRunning: (running: boolean) => void; +}) { + const { createNewTerminal, terminals, closeTerminal } = useTerminal(); + const { setIsPreviewCollapsed, previewPanelRef} = usePreview(); + + const handleRun = () => { + if (isRunning) { + console.log('Stopping sandbox...'); + console.log('Closing Preview Window'); + + terminals.forEach(term => { + if (term.terminal) { + closeTerminal(term.id); + console.log('Closing Terminal', term.id); + } + }); + + setIsPreviewCollapsed(true); + previewPanelRef.current?.collapse(); + } else { + console.log('Running sandbox...'); + console.log('Opening Terminal'); + console.log('Opening Preview Window'); + + if (terminals.length < 4) { + createNewTerminal(); + } else { + toast.error("You reached the maximum # of terminals."); + console.error('Maximum number of terminals reached.'); + } + + setIsPreviewCollapsed(false); + previewPanelRef.current?.expand(); + } + setIsRunning(!isRunning); + }; + + return ( + <> + + + ); +} \ No newline at end of file diff --git a/frontend/components/editor/preview/index.tsx b/frontend/components/editor/preview/index.tsx index a400d14..940f8ba 100644 --- a/frontend/components/editor/preview/index.tsx +++ b/frontend/components/editor/preview/index.tsx @@ -1,13 +1,9 @@ "use client" import { - ChevronLeft, - ChevronRight, - Globe, Link, RotateCw, TerminalSquare, - UnfoldVertical, } from "lucide-react" import { useRef, useState } from "react" import { toast } from "sonner" @@ -27,22 +23,22 @@ export default function PreviewWindow({ return ( <>
Preview
{collapsed ? ( - - + { }}> + ) : ( <> - {/* Todo, make this open inspector */} - {/* {}}> - + {/* Removed the unfoldvertical button since we have the same thing via the run button. + + + */} {children} diff --git a/frontend/components/editor/terminals/index.tsx b/frontend/components/editor/terminals/index.tsx index a777eeb..3381b33 100644 --- a/frontend/components/editor/terminals/index.tsx +++ b/frontend/components/editor/terminals/index.tsx @@ -2,55 +2,62 @@ import { Button } from "@/components/ui/button"; import Tab from "@/components/ui/tab"; -import { closeTerminal, createTerminal } from "@/lib/terminal"; import { Terminal } from "@xterm/xterm"; import { Loader2, Plus, SquareTerminal, TerminalSquare } from "lucide-react"; -import { Socket } from "socket.io-client"; import { toast } from "sonner"; import EditorTerminal from "./terminal"; -import { useState } from "react"; +import { useTerminal } from "@/context/TerminalContext"; +import { useEffect } from "react"; + +export default function Terminals() { + const { + terminals, + setTerminals, + socket, + createNewTerminal, + closeTerminal, + activeTerminalId, + setActiveTerminalId, + creatingTerminal, + } = useTerminal(); -export default function Terminals({ - terminals, - setTerminals, - socket, -}: { - terminals: { id: string; terminal: Terminal | null }[]; - setTerminals: React.Dispatch< - React.SetStateAction< - { - id: string; - terminal: Terminal | null; - }[] - > - >; - socket: Socket; -}) { - const [activeTerminalId, setActiveTerminalId] = useState(""); - const [creatingTerminal, setCreatingTerminal] = useState(false); - const [closingTerminal, setClosingTerminal] = useState(""); const activeTerminal = terminals.find((t) => t.id === activeTerminalId); + // Effect to set the active terminal when a new one is created + useEffect(() => { + if (terminals.length > 0 && !activeTerminalId) { + setActiveTerminalId(terminals[terminals.length - 1].id); + } + }, [terminals, activeTerminalId, setActiveTerminalId]); + + const handleCreateTerminal = () => { + if (terminals.length >= 4) { + toast.error("You reached the maximum # of terminals."); + return; + } + createNewTerminal(); + }; + + const handleCloseTerminal = (termId: string) => { + closeTerminal(termId); + if (activeTerminalId === termId) { + const remainingTerminals = terminals.filter(t => t.id !== termId); + if (remainingTerminals.length > 0) { + setActiveTerminalId(remainingTerminals[0].id); + } else { + setActiveTerminalId(""); + } + } + }; + return ( <>
{terminals.map((term) => ( setActiveTerminalId(term.id)} - onClose={() => - closeTerminal({ - term, - terminals, - setTerminals, - setActiveTerminalId, - setClosingTerminal, - socket, - activeTerminalId, - }) - } - closing={closingTerminal === term.id} + onClose={() => handleCloseTerminal(term.id)} selected={activeTerminalId === term.id} > @@ -59,18 +66,7 @@ export default function Terminals({ ))} + + ); +} \ No newline at end of file From 870783940da4395037ea066e10db200176409ecf Mon Sep 17 00:00:00 2001 From: James Murdza Date: Tue, 23 Jul 2024 17:54:44 -0400 Subject: [PATCH 22/29] Add Dokku environment variables to .env.example. --- backend/server/.env.example | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/server/.env.example b/backend/server/.env.example index 594efc5..e9e502a 100644 --- a/backend/server/.env.example +++ b/backend/server/.env.example @@ -1,8 +1,13 @@ # Set WORKERS_KEY to be the same as KEY in /backend/storage/wrangler.toml. # Set DATABASE_WORKER_URL and STORAGE_WORKER_URL after deploying the workers. +# DOKKU_HOST and DOKKU_USERNAME are used to authenticate via SSH with the Dokku server +# DOKKU_KEY is the path to an SSH (.pem) key on the local machine PORT=4000 WORKERS_KEY= DATABASE_WORKER_URL= STORAGE_WORKER_URL= -E2B_API_KEY= \ No newline at end of file +E2B_API_KEY= +DOKKU_HOST= +DOKKU_USERNAME= +DOKKU_KEY= \ No newline at end of file From 74a43523231e9f1018a9c42f9b69fb0f2848bdf4 Mon Sep 17 00:00:00 2001 From: Akhilesh Rangani Date: Tue, 23 Jul 2024 20:17:50 -0400 Subject: [PATCH 23/29] fix: added terminal response handling --- .../components/editor/terminals/index.tsx | 23 +++++-------------- .../components/editor/terminals/terminal.tsx | 14 +++++++++++ frontend/context/TerminalContext.tsx | 2 +- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/frontend/components/editor/terminals/index.tsx b/frontend/components/editor/terminals/index.tsx index 3381b33..076f121 100644 --- a/frontend/components/editor/terminals/index.tsx +++ b/frontend/components/editor/terminals/index.tsx @@ -38,26 +38,15 @@ export default function Terminals() { createNewTerminal(); }; - const handleCloseTerminal = (termId: string) => { - closeTerminal(termId); - if (activeTerminalId === termId) { - const remainingTerminals = terminals.filter(t => t.id !== termId); - if (remainingTerminals.length > 0) { - setActiveTerminalId(remainingTerminals[0].id); - } else { - setActiveTerminalId(""); - } - } - }; - return ( <>
{terminals.map((term) => ( setActiveTerminalId(term.id)} - onClose={() => handleCloseTerminal(term.id)} + onClose={() => closeTerminal(term.id)} selected={activeTerminalId === term.id} > @@ -88,10 +77,10 @@ export default function Terminals() { term={term.terminal} setTerm={(t: Terminal) => { setTerminals((prev) => - prev.map((prevTerm) => - prevTerm.id === term.id - ? { ...prevTerm, terminal: t } - : prevTerm + prev.map((term) => + term.id === activeTerminalId + ? { ...term, terminal: t } + : term ) ); }} diff --git a/frontend/components/editor/terminals/terminal.tsx b/frontend/components/editor/terminals/terminal.tsx index 3acbe23..3399159 100644 --- a/frontend/components/editor/terminals/terminal.tsx +++ b/frontend/components/editor/terminals/terminal.tsx @@ -74,6 +74,20 @@ export default function EditorTerminal({ }; }, [term, terminalRef.current]); + useEffect(() => { + if (!term) return; + const handleTerminalResponse = (response: { id: string; data: string }) => { + if (response.id === id) { + term.write(response.data); + } + }; + socket.on("terminalResponse", handleTerminalResponse); + + return () => { + socket.off("terminalResponse", handleTerminalResponse); + }; + }, [term, id, socket]); + return ( <>
Date: Tue, 23 Jul 2024 22:17:26 -0400 Subject: [PATCH 24/29] Deploy projects by pushing files to Dokku server via git. --- backend/server/package-lock.json | 70 +++++++++++++++++++++++ backend/server/package.json | 1 + backend/server/src/SecureGitClient.ts | 80 +++++++++++++++++++++++++++ backend/server/src/index.ts | 36 +++++++++++- 4 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 backend/server/src/SecureGitClient.ts diff --git a/backend/server/package-lock.json b/backend/server/package-lock.json index d167660..288efe9 100644 --- a/backend/server/package-lock.json +++ b/backend/server/package-lock.json @@ -15,6 +15,7 @@ "e2b": "^0.16.1", "express": "^4.19.2", "rate-limiter-flexible": "^5.0.3", + "simple-git": "^3.25.0", "socket.io": "^4.7.5", "ssh2": "^1.15.0", "zod": "^3.22.4" @@ -77,6 +78,40 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@kwsites/file-exists": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", + "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", + "dependencies": { + "debug": "^4.1.1" + } + }, + "node_modules/@kwsites/file-exists/node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@kwsites/file-exists/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/@kwsites/promise-deferred": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", + "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==" + }, "node_modules/@socket.io/component-emitter": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.1.tgz", @@ -1695,6 +1730,41 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-git": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.25.0.tgz", + "integrity": "sha512-KIY5sBnzc4yEcJXW7Tdv4viEz8KyG+nU0hay+DWZasvdFOYKeUZ6Xc25LUHHjw0tinPT7O1eY6pzX7pRT1K8rw==", + "dependencies": { + "@kwsites/file-exists": "^1.1.1", + "@kwsites/promise-deferred": "^1.1.1", + "debug": "^4.3.5" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/git-js?sponsor=1" + } + }, + "node_modules/simple-git/node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/simple-git/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/simple-update-notifier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", diff --git a/backend/server/package.json b/backend/server/package.json index dc0f09c..19e5ee5 100644 --- a/backend/server/package.json +++ b/backend/server/package.json @@ -17,6 +17,7 @@ "e2b": "^0.16.1", "express": "^4.19.2", "rate-limiter-flexible": "^5.0.3", + "simple-git": "^3.25.0", "socket.io": "^4.7.5", "ssh2": "^1.15.0", "zod": "^3.22.4" diff --git a/backend/server/src/SecureGitClient.ts b/backend/server/src/SecureGitClient.ts new file mode 100644 index 0000000..5c115b8 --- /dev/null +++ b/backend/server/src/SecureGitClient.ts @@ -0,0 +1,80 @@ +import simpleGit, { SimpleGit } from "simple-git"; +import path from "path"; +import fs from "fs"; +import os from "os"; + +export type FileData = { + id: string; + data: string; +}; + +export class SecureGitClient { + private gitUrl: string; + private sshKeyPath: string; + + constructor(gitUrl: string, sshKeyPath: string) { + this.gitUrl = gitUrl; + this.sshKeyPath = sshKeyPath; + } + + async pushFiles(fileData: FileData[], repository: string): Promise { + let tempDir: string | undefined; + + try { + // Create a temporary directory + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'git-push-')); + console.log(`Temporary directory created: ${tempDir}`); + + // Write files to the temporary directory + for (const { id, data } of fileData) { + const filePath = path.join(tempDir, id); + const dirPath = path.dirname(filePath); + + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } + + console.log("Writing ", filePath, data); + fs.writeFileSync(filePath, data); + } + + // Initialize the simple-git instance with the temporary directory and custom SSH command + const git: SimpleGit = simpleGit(tempDir, { + config: [ + 'core.sshCommand=ssh -i ' + this.sshKeyPath + ' -o IdentitiesOnly=yes' + ] + }); + + // Initialize a new Git repository + await git.init(); + + // Add remote repository + await git.addRemote("origin", `${this.gitUrl}:${repository}`); + + // Add files to the repository + for (const {id, data} of fileData) { + await git.add(id); + } + + // Commit the changes + await git.commit("Add files."); + + // Push the changes to the remote repository + await git.push("origin", "master"); + + console.log("Files successfully pushed to the repository"); + + if (tempDir) { + fs.rmSync(tempDir, { recursive: true, force: true }); + console.log(`Temporary directory removed: ${tempDir}`); + } + } catch (error) { + if (tempDir) { + fs.rmSync(tempDir, { recursive: true, force: true }); + console.log(`Temporary directory removed: ${tempDir}`); + } + console.error("Error pushing files to the repository:", error); + throw error; + } + } +} \ No newline at end of file diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 187db67..9a2ac0d 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -1,11 +1,11 @@ -import os from "os"; import path from "path"; import cors from "cors"; import express, { Express } from "express"; import dotenv from "dotenv"; import { createServer } from "http"; import { Server } from "socket.io"; -import { DokkuClient, SSHConfig } from "./DokkuClient"; +import { DokkuClient } from "./DokkuClient"; +import { SecureGitClient, FileData } from "./SecureGitClient"; import fs from "fs"; import { z } from "zod"; @@ -126,6 +126,11 @@ const client = new DokkuClient({ client.connect(); +const git = new SecureGitClient( + "dokku@gitwit.app", + process.env.DOKKU_KEY +) + io.on("connection", async (socket) => { try { if (inactivityTimeout) clearTimeout(inactivityTimeout); @@ -292,6 +297,33 @@ io.on("connection", async (socket) => { } ); + socket.on( + "deploy", + async (callback: (response: CallbackResponse) => void) => { + try { + // Push the project files to the Dokku server + console.log("Deploying project ${data.sandboxId}..."); + // Remove the /project/[id]/ component of each file path: + const fixedFilePaths = sandboxFiles.fileData.map((file) => { + return { + ...file, + id: file.id.split("/").slice(2).join("/"), + }; + }); + // Push all files to Dokku. + await git.pushFiles(fixedFilePaths, data.sandboxId); + callback({ + success: true, + }); + } catch (error) { + callback({ + success: false, + message: "Failed to deploy project: " + error, + }); + } + } + ); + socket.on("createFile", async (name: string, callback) => { try { const size: number = await getProjectSize(data.sandboxId); From 02ea851fb72cbc8c520520a051a1ed92a9228f58 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Tue, 23 Jul 2024 22:17:36 -0400 Subject: [PATCH 25/29] Add deploy test. --- tests/index.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/index.ts b/tests/index.ts index e2fc1f4..a36868c 100644 --- a/tests/index.ts +++ b/tests/index.ts @@ -11,7 +11,7 @@ interface CallbackResponse { } let socketRef: Socket = io( - `http://localhost:4000?userId=user_2hFB6KcK6bb3Gx9241UXsxFq4kO&sandboxId=aabuk4vneecj2csni24kpabv`, + `http://localhost:4000?userId=user_2hFB6KcK6bb3Gx9241UXsxFq4kO&sandboxId=v30a2c48xal03tzio7mapt19`, { timeout: 2000, } @@ -28,6 +28,14 @@ socketRef.on("connect", async () => { console.log("Error:", response.message); } }); + + socketRef.emit("deploy", (response: CallbackResponse) => { + if (response.success) { + console.log("It worked!"); + } else { + console.log("Error:", response.message); + } + }); }); socketRef.on("disconnect", () => { From a74f7bf71a511b94312ca5e1e6ac471147432d5e Mon Sep 17 00:00:00 2001 From: James Murdza Date: Wed, 31 Jul 2024 17:09:24 -0700 Subject: [PATCH 26/29] Change React template from Vite to create-react-app. --- backend/server/src/index.ts | 2 +- backend/storage/src/startercode.ts | 62 +++++++++++++++--------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 9a2ac0d..4463d2e 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -158,7 +158,7 @@ io.on("connection", async (socket) => { console.log("Created container ", data.sandboxId); io.emit( "previewURL", - "https://" + containers[data.sandboxId].getHostname(5173) + "https://" + containers[data.sandboxId].getHostname(3000) ); } } catch (e: any) { diff --git a/backend/storage/src/startercode.ts b/backend/storage/src/startercode.ts index 3979f9b..ae768e1 100644 --- a/backend/storage/src/startercode.ts +++ b/backend/storage/src/startercode.ts @@ -21,49 +21,49 @@ const startercode = { { name: "package.json", body: `{ - "name": "react", + "name": "react-app", + "version": "0.1.0", "private": true, - "version": "0.0.0", - "type": "module", - "scripts": { - "dev": "vite", - "build": "vite build", - "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0", - "preview": "vite preview" - }, "dependencies": { "react": "^18.2.0", - "react-dom": "^18.2.0" + "react-dom": "^18.2.0", + "react-scripts": "5.0.1" + }, + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test", + "eject": "react-scripts eject" + }, + "eslintConfig": { + "extends": [ + "react-app", + "react-app/jest" + ] + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] }, "devDependencies": { "@types/react": "^18.2.66", "@types/react-dom": "^18.2.22", - "@vitejs/plugin-react": "^4.2.1", "eslint": "^8.57.0", "eslint-plugin-react": "^7.34.1", - "eslint-plugin-react-hooks": "^4.6.0", - "eslint-plugin-react-refresh": "^0.4.6", - "vite": "^5.2.0" + "eslint-plugin-react-hooks": "^4.6.0" } }`, }, { - name: "vite.config.js", - body: `import { defineConfig } from 'vite' -import react from '@vitejs/plugin-react' - -// https://vitejs.dev/config/ -export default defineConfig({ - plugins: [react()], - server: { - port: 5173, - host: "0.0.0.0", - } -}) -`, - }, - { - name: "index.html", + name: "public/index.html", body: ` @@ -133,7 +133,7 @@ export default App `, }, { - name: "src/main.jsx", + name: "src/index.js", body: `import React from 'react' import ReactDOM from 'react-dom/client' import App from './App.jsx' From 6a31161c0ad9dcc5ec99e3fa04cf81d92446825c Mon Sep 17 00:00:00 2001 From: James Murdza Date: Wed, 31 Jul 2024 17:49:59 -0700 Subject: [PATCH 27/29] Start development server when run button is clicked. --- frontend/components/editor/navbar/run.tsx | 2 +- frontend/context/TerminalContext.tsx | 5 +++-- frontend/lib/terminal.ts | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/components/editor/navbar/run.tsx b/frontend/components/editor/navbar/run.tsx index 688886f..8e5a3b2 100644 --- a/frontend/components/editor/navbar/run.tsx +++ b/frontend/components/editor/navbar/run.tsx @@ -36,7 +36,7 @@ export default function RunButtonModal({ console.log('Opening Preview Window'); if (terminals.length < 4) { - createNewTerminal(); + createNewTerminal("yarn install && yarn start"); } else { toast.error("You reached the maximum # of terminals."); console.error('Maximum number of terminals reached.'); diff --git a/frontend/context/TerminalContext.tsx b/frontend/context/TerminalContext.tsx index 517a355..573a32e 100644 --- a/frontend/context/TerminalContext.tsx +++ b/frontend/context/TerminalContext.tsx @@ -13,7 +13,7 @@ interface TerminalContextType { setActiveTerminalId: React.Dispatch>; creatingTerminal: boolean; setCreatingTerminal: React.Dispatch>; - createNewTerminal: () => void; + createNewTerminal: (command?: string) => Promise; closeTerminal: (id: string) => void; setUserAndSandboxId: (userId: string, sandboxId: string) => void; } @@ -50,7 +50,7 @@ export const TerminalProvider: React.FC<{ children: React.ReactNode }> = ({ chil } }, [userId, sandboxId]); - const createNewTerminal = async () => { + const createNewTerminal = async (command?: string): Promise => { if (!socket) return; setCreatingTerminal(true); try { @@ -58,6 +58,7 @@ export const TerminalProvider: React.FC<{ children: React.ReactNode }> = ({ chil setTerminals, setActiveTerminalId, setCreatingTerminal, + command, socket, }); } catch (error) { diff --git a/frontend/lib/terminal.ts b/frontend/lib/terminal.ts index fb3200f..6f45ee8 100644 --- a/frontend/lib/terminal.ts +++ b/frontend/lib/terminal.ts @@ -8,6 +8,7 @@ export const createTerminal = ({ setTerminals, setActiveTerminalId, setCreatingTerminal, + command, socket, }: { setTerminals: React.Dispatch>; setActiveTerminalId: React.Dispatch>; setCreatingTerminal: React.Dispatch>; + command?: string; socket: Socket; }) => { @@ -29,6 +31,7 @@ export const createTerminal = ({ setTimeout(() => { socket.emit("createTerminal", id, () => { setCreatingTerminal(false); + if (command) socket.emit("terminalData", id, command + "\n"); }); }, 1000); }; From 6c615f1a4f2a714891e52b6b3b3b92452cb6ddc3 Mon Sep 17 00:00:00 2001 From: James Murdza Date: Wed, 31 Jul 2024 18:16:04 -0700 Subject: [PATCH 28/29] Detect running server port number from terminal output. --- backend/server/src/index.ts | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/server/src/index.ts b/backend/server/src/index.ts index 4463d2e..a248909 100644 --- a/backend/server/src/index.ts +++ b/backend/server/src/index.ts @@ -156,10 +156,6 @@ io.on("connection", async (socket) => { if (!containers[data.sandboxId]) { containers[data.sandboxId] = await Sandbox.create(); console.log("Created container ", data.sandboxId); - io.emit( - "previewURL", - "https://" + containers[data.sandboxId].getHostname(3000) - ); } } catch (e: any) { console.error(`Error creating container ${data.sandboxId}:`, e); @@ -492,8 +488,26 @@ io.on("connection", async (socket) => { await lockManager.acquireLock(data.sandboxId, async () => { try { terminals[id] = await containers[data.sandboxId].terminal.start({ - onData: (data: string) => { - io.emit("terminalResponse", { id, data }); + onData: (responseData: string) => { + io.emit("terminalResponse", { id, data: responseData }); + + function extractPortNumber(inputString: string) { + // Remove ANSI escape codes + const cleanedString = inputString.replace(/\x1B\[[0-9;]*m/g, ''); + // Regular expression to match port number + const regex = /http:\/\/localhost:(\d+)/; + // If a match is found, return the port number + const match = cleanedString.match(regex); + return match ? match[1] : null; + } + const port = parseInt(extractPortNumber(responseData) ?? ""); + if (port) { + io.emit( + "previewURL", + "https://" + containers[data.sandboxId].getHostname(port) + ); + } + }, size: { cols: 80, rows: 20 }, onExit: () => console.log("Terminal exited", id), From d0a9c8548c47f033d5c2a2a02ffefdcf071e6b7f Mon Sep 17 00:00:00 2001 From: James Murdza Date: Wed, 31 Jul 2024 18:17:01 -0700 Subject: [PATCH 29/29] Remove unecessary logging. --- frontend/components/editor/terminals/terminal.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/components/editor/terminals/terminal.tsx b/frontend/components/editor/terminals/terminal.tsx index 3399159..1187acb 100644 --- a/frontend/components/editor/terminals/terminal.tsx +++ b/frontend/components/editor/terminals/terminal.tsx @@ -55,7 +55,6 @@ export default function EditorTerminal({ fitAddon.fit(); const disposableOnData = term.onData((data) => { - console.log("terminalData", id, data); socket.emit("terminalData", id, data); });