From ae38a77759d6c9668c9a6b88c43e54af45b8ed6c Mon Sep 17 00:00:00 2001 From: James Murdza Date: Sat, 19 Oct 2024 16:23:31 -0600 Subject: [PATCH] chore: refactor into RemoteFileStorage --- backend/server/src/FileManager.ts | 89 ++++++++++--- backend/server/src/RemoteFileStorage.ts | 117 ++++++++++++++++ backend/server/src/fileoperations.ts | 170 ------------------------ 3 files changed, 187 insertions(+), 189 deletions(-) create mode 100644 backend/server/src/RemoteFileStorage.ts delete mode 100644 backend/server/src/fileoperations.ts diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index 2ac1a80..d615c43 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -1,14 +1,6 @@ import { FilesystemEvent, Sandbox, WatchHandle } from "e2b" import path from "path" -import { - createFile, - deleteFile, - getFolder, - getProjectSize, - getSandboxFiles, - renameFile, - saveFile, -} from "./fileoperations" +import RemoteFileStorage from "./RemoteFileStorage" import { MAX_BODY_SIZE } from "./ratelimit" import { TFile, TFileData, TFolder } from "./types" @@ -18,6 +10,65 @@ export type SandboxFiles = { fileData: TFileData[] } +const processFiles = async (paths: string[], id: string) => { + const root: TFolder = { id: "/", type: "folder", name: "/", children: [] } + const fileData: TFileData[] = [] + + paths.forEach((path) => { + const allParts = path.split("/") + if (allParts[1] !== id) { + return + } + + const parts = allParts.slice(2) + let current: TFolder = root + + for (let i = 0; i < parts.length; i++) { + const part = parts[i] + const isFile = i === parts.length - 1 && part.length + const existing = current.children.find((child) => child.name === part) + + if (existing) { + if (!isFile) { + current = existing as TFolder + } + } else { + if (isFile) { + const file: TFile = { id: path, type: "file", name: part } + current.children.push(file) + fileData.push({ id: path, data: "" }) + } else { + const folder: TFolder = { + // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css + id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, + type: "folder", + name: part, + children: [], + } + current.children.push(folder) + current = folder + } + } + } + }) + + await Promise.all( + fileData.map(async (file) => { + const data = await RemoteFileStorage.fetchFileContent(file.id) + file.data = data + }) + ) + + return { + files: root.children, + fileData, + } +} + +const getSandboxFiles = async (id: string) => { + return await processFiles(await RemoteFileStorage.getSandboxPaths(id), id) +} + // FileManager class to handle file operations in a sandbox export class FileManager { private sandboxId: string @@ -285,7 +336,7 @@ export class FileManager { // Get folder content async getFolder(folderId: string): Promise { - return getFolder(folderId) + return RemoteFileStorage.getFolder(folderId) } // Save file content @@ -295,7 +346,7 @@ export class FileManager { if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) { throw new Error("File size too large. Please reduce the file size.") } - await saveFile(fileId, body) + await RemoteFileStorage.saveFile(fileId, body) const file = this.sandboxFiles.fileData.find((f) => f.id === fileId) if (!file) return file.data = body @@ -323,7 +374,7 @@ export class FileManager { fileData.id = newFileId file.id = newFileId - await renameFile(fileId, newFileId, fileData.data) + await RemoteFileStorage.renameFile(fileId, newFileId, fileData.data) const newFiles = await getSandboxFiles(this.sandboxId) return newFiles.files } @@ -346,7 +397,7 @@ export class FileManager { // Create a new file async createFile(name: string): Promise { - const size: number = await getProjectSize(this.sandboxId) + const size: number = await RemoteFileStorage.getProjectSize(this.sandboxId) if (size > 200 * 1024 * 1024) { throw new Error("Project size exceeded. Please delete some files.") } @@ -367,7 +418,7 @@ export class FileManager { data: "", }) - await createFile(id) + await RemoteFileStorage.createFile(id) return true } @@ -389,7 +440,7 @@ export class FileManager { await this.moveFileInContainer(fileId, newFileId) await this.fixPermissions() - await renameFile(fileId, newFileId, fileData.data) + await RemoteFileStorage.renameFile(fileId, newFileId, fileData.data) fileData.id = newFileId file.id = newFileId @@ -405,7 +456,7 @@ export class FileManager { (f) => f.id !== fileId ) - await deleteFile(fileId) + await RemoteFileStorage.deleteFile(fileId) const newFiles = await getSandboxFiles(this.sandboxId) return newFiles.files @@ -413,7 +464,7 @@ export class FileManager { // Delete a folder async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> { - const files = await getFolder(folderId) + const files = await RemoteFileStorage.getFolder(folderId) await Promise.all( files.map(async (file) => { @@ -421,7 +472,7 @@ export class FileManager { this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter( (f) => f.id !== file ) - await deleteFile(file) + await RemoteFileStorage.deleteFile(file) }) ) @@ -437,4 +488,4 @@ export class FileManager { }) ) } -} \ No newline at end of file +} diff --git a/backend/server/src/RemoteFileStorage.ts b/backend/server/src/RemoteFileStorage.ts new file mode 100644 index 0000000..e5ed4b2 --- /dev/null +++ b/backend/server/src/RemoteFileStorage.ts @@ -0,0 +1,117 @@ +import * as dotenv from "dotenv" +import { R2Files } from "./types" + +dotenv.config() + +export const RemoteFileStorage = { + getSandboxPaths: async (id: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + const data: R2Files = await res.json() + + return data.objects.map((obj) => obj.key) + }, + + getFolder: async (folderId: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + const data: R2Files = await res.json() + + return data.objects.map((obj) => obj.key) + }, + + fetchFileContent: async (fileId: string): Promise => { + try { + const fileRes = await fetch( + `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + return await fileRes.text() + } catch (error) { + console.error("ERROR fetching file:", error) + return "" + } + }, + + createFile: async (fileId: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId }), + }) + return res.ok + }, + + renameFile: async ( + fileId: string, + newFileId: string, + data: string + ) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId, newFileId, data }), + }) + return res.ok + }, + + saveFile: async (fileId: string, data: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId, data }), + }) + return res.ok + }, + + deleteFile: async (fileId: string) => { + const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { + method: "DELETE", + headers: { + "Content-Type": "application/json", + Authorization: `${process.env.WORKERS_KEY}`, + }, + body: JSON.stringify({ fileId }), + }) + return res.ok + }, + + getProjectSize: async (id: string) => { + const res = await fetch( + `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`, + { + headers: { + Authorization: `${process.env.WORKERS_KEY}`, + }, + } + ) + return (await res.json()).size + } +} + +export default RemoteFileStorage \ No newline at end of file diff --git a/backend/server/src/fileoperations.ts b/backend/server/src/fileoperations.ts deleted file mode 100644 index 1157487..0000000 --- a/backend/server/src/fileoperations.ts +++ /dev/null @@ -1,170 +0,0 @@ -import * as dotenv from "dotenv" -import { R2Files, TFile, TFileData, TFolder } from "./types" - -dotenv.config() - -export const getSandboxFiles = async (id: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - const data: R2Files = await res.json() - - const paths = data.objects.map((obj) => obj.key) - const processedFiles = await processFiles(paths, id) - return processedFiles -} - -export const getFolder = async (folderId: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - const data: R2Files = await res.json() - - return data.objects.map((obj) => obj.key) -} - -const processFiles = async (paths: string[], id: string) => { - const root: TFolder = { id: "/", type: "folder", name: "/", children: [] } - const fileData: TFileData[] = [] - - paths.forEach((path) => { - const allParts = path.split("/") - if (allParts[1] !== id) { - return - } - - const parts = allParts.slice(2) - let current: TFolder = root - - for (let i = 0; i < parts.length; i++) { - const part = parts[i] - const isFile = i === parts.length - 1 && part.length - const existing = current.children.find((child) => child.name === part) - - if (existing) { - if (!isFile) { - current = existing as TFolder - } - } else { - if (isFile) { - const file: TFile = { id: path, type: "file", name: part } - current.children.push(file) - fileData.push({ id: path, data: "" }) - } else { - const folder: TFolder = { - // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css - id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`, - type: "folder", - name: part, - children: [], - } - current.children.push(folder) - current = folder - } - } - } - }) - - await Promise.all( - fileData.map(async (file) => { - const data = await fetchFileContent(file.id) - file.data = data - }) - ) - - return { - files: root.children, - fileData, - } -} - -const fetchFileContent = async (fileId: string): Promise => { - try { - const fileRes = await fetch( - `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - return await fileRes.text() - } catch (error) { - console.error("ERROR fetching file:", error) - return "" - } -} - -export const createFile = async (fileId: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId }), - }) - return res.ok -} - -export const renameFile = async ( - fileId: string, - newFileId: string, - data: string -) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId, newFileId, data }), - }) - return res.ok -} - -export const saveFile = async (fileId: string, data: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId, data }), - }) - return res.ok -} - -export const deleteFile = async (fileId: string) => { - const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, { - method: "DELETE", - headers: { - "Content-Type": "application/json", - Authorization: `${process.env.WORKERS_KEY}`, - }, - body: JSON.stringify({ fileId }), - }) - return res.ok -} - -export const getProjectSize = async (id: string) => { - const res = await fetch( - `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`, - { - headers: { - Authorization: `${process.env.WORKERS_KEY}`, - }, - } - ) - return (await res.json()).size -}