From 10104c31b91ab6178f563c3af6515921e7bcdf0b Mon Sep 17 00:00:00 2001 From: omar rashed Date: Sat, 16 Nov 2024 21:48:40 -0500 Subject: [PATCH] fix: prepare zip file on the backend when exporting a project --- backend/server/package-lock.json | 116 ++++++++++++++++++ backend/server/package.json | 4 +- backend/server/src/FileManager.ts | 62 ++++++++++ backend/server/src/Sandbox.ts | 11 +- .../editor/navbar/downloadButton.tsx | 43 ++++--- 5 files changed, 210 insertions(+), 26 deletions(-) diff --git a/backend/server/package-lock.json b/backend/server/package-lock.json index dd284cb..dab4e32 100644 --- a/backend/server/package-lock.json +++ b/backend/server/package-lock.json @@ -14,6 +14,7 @@ "dotenv": "^16.4.5", "e2b": "^0.16.2-beta.47", "express": "^4.19.2", + "jzip": "^1.0.0", "rate-limiter-flexible": "^5.0.3", "simple-git": "^3.25.0", "socket.io": "^4.7.5", @@ -23,6 +24,7 @@ "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^4.17.21", + "@types/jszip": "^3.4.1", "@types/node": "^20.12.7", "@types/ssh2": "^1.15.0", "nodemon": "^3.1.0", @@ -225,6 +227,16 @@ "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", "dev": true }, + "node_modules/@types/jszip": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@types/jszip/-/jszip-3.4.1.tgz", + "integrity": "sha512-TezXjmf3lj+zQ651r6hPqvSScqBLvyPI9FxdXBqpEwBijNGQ2NXpaFW/7joGzveYkKQUil7iiDHLo6LV71Pc0A==", + "deprecated": "This is a stub types definition. jszip provides its own type definitions, so you do not need this installed.", + "dev": true, + "dependencies": { + "jszip": "*" + } + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -656,6 +668,12 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, "node_modules/cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -1156,6 +1174,12 @@ "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", "dev": true }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -1219,6 +1243,38 @@ "node": ">=0.12.0" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dev": true, + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jzip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/jzip/-/jzip-1.0.0.tgz", + "integrity": "sha512-pyDHf5zvxE5DC47ftNff2AU3UdJe0TYSFki0Ji6GapuZC7p2EizIbPHV7dkLj43RPv2Vj3p1xwC11kDv6dyCrA==" + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -1448,6 +1504,12 @@ "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.0.8.tgz", "integrity": "sha512-1eNjQtbfNi5Z/kFhagDIaIRj6qqDzhjNJKz8cmMW0CVdGwT6e1GLbAfgI0d28VTJa1A8jz82jm/4dG8qNoNS8g==" }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1478,6 +1540,12 @@ "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==" }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1537,6 +1605,27 @@ "node": ">= 0.8" } }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -1664,6 +1753,12 @@ "node": ">= 0.4" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -1892,6 +1987,21 @@ "node": ">= 0.8" } }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -2078,6 +2188,12 @@ "node": ">=6.14.2" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/backend/server/package.json b/backend/server/package.json index 435cd1b..ff093dc 100644 --- a/backend/server/package.json +++ b/backend/server/package.json @@ -16,6 +16,7 @@ "dotenv": "^16.4.5", "e2b": "^0.16.2-beta.47", "express": "^4.19.2", + "jzip": "^1.0.0", "rate-limiter-flexible": "^5.0.3", "simple-git": "^3.25.0", "socket.io": "^4.7.5", @@ -25,10 +26,11 @@ "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^4.17.21", + "@types/jszip": "^3.4.1", "@types/node": "^20.12.7", "@types/ssh2": "^1.15.0", "nodemon": "^3.1.0", "ts-node": "^10.9.2", "typescript": "^5.4.5" } -} \ No newline at end of file +} diff --git a/backend/server/src/FileManager.ts b/backend/server/src/FileManager.ts index bbb61ce..0022c1f 100644 --- a/backend/server/src/FileManager.ts +++ b/backend/server/src/FileManager.ts @@ -1,4 +1,5 @@ import { FilesystemEvent, Sandbox, WatchHandle } from "e2b" +import JSZip from "jszip" import path from "path" import RemoteFileStorage from "./RemoteFileStorage" import { MAX_BODY_SIZE } from "./ratelimit" @@ -472,6 +473,67 @@ export class FileManager { return true } + public async getFilesForDownload(): Promise { + // Get all file paths, excluding node_modules + const result = await this.sandbox.commands.run( + `find "${this.dirName}" -path "${this.dirName}/node_modules" -prune -o -type f -print` + ) + const filePaths = result.stdout.split("\n").filter((path) => path) + + if (!filePaths || filePaths.length === 0) { + console.error( + "No files found in the sandbox project directory for download." + ) + return "" + } + + console.log("Paths found for download (excluding node_modules):", filePaths) + + // Create new JSZip instance + const zip = new JSZip() + + // Add files to zip with synchronized content + for (const filePath of filePaths) { + const relativePath = filePath.replace(this.dirName, "") // Remove base directory from path + try { + // Get the latest content directly from the sandbox + let content = await this.sandbox.files.read(filePath) + + // Update the fileData cache with the latest content + const fileDataEntry = this.fileData.find((f) => f.id === relativePath) + if (fileDataEntry) { + fileDataEntry.data = typeof content === "string" ? content : "" + } + + // Set content to an empty string if file is empty or undefined + if (typeof content !== "string") { + content = "" + } + + zip.file(relativePath, content) + console.log(`Added file to ZIP: ${relativePath}`) + } catch (error) { + console.error(`Failed to read content for ${relativePath}:`, error) + } + } + + // Generate zip file + const zipBlob = await zip.generateAsync({ + type: "blob", + compression: "DEFLATE", + compressionOptions: { + level: 6, + }, + }) + + // Convert Blob to Base64 + const zipBlobArrayBuffer = await zipBlob.arrayBuffer() + const zipBlobBase64 = btoa( + String.fromCharCode(...new Uint8Array(zipBlobArrayBuffer)) + ) + + return zipBlobBase64 + } // Create a new folder async createFolder(name: string): Promise { diff --git a/backend/server/src/Sandbox.ts b/backend/server/src/Sandbox.ts index 4657523..f81d483 100644 --- a/backend/server/src/Sandbox.ts +++ b/backend/server/src/Sandbox.ts @@ -13,7 +13,7 @@ import { } from "./ratelimit" import { SecureGitClient } from "./SecureGitClient" import { TerminalManager } from "./TerminalManager" -import { TFile, TFileData, TFolder } from "./types" +import { TFile, TFolder } from "./types" import { LockManager } from "./utils" const lockManager = new LockManager() @@ -251,13 +251,10 @@ export class Sandbox { const handleDownloadFiles: SocketHandler = async () => { if (!this.fileManager) throw Error("No file manager") - // Get all files with their data through fileManager - const files = this.fileManager.fileData.map((file: TFileData) => ({ - path: file.id.startsWith("/") ? file.id.slice(1) : file.id, - content: file.data, - })) + // Get the Base64 encoded ZIP string + const zipBase64 = await this.fileManager.getFilesForDownload() - return { files } + return { zipBlob: zipBase64 } } return { diff --git a/frontend/components/editor/navbar/downloadButton.tsx b/frontend/components/editor/navbar/downloadButton.tsx index c036ffc..6a58511 100644 --- a/frontend/components/editor/navbar/downloadButton.tsx +++ b/frontend/components/editor/navbar/downloadButton.tsx @@ -1,29 +1,36 @@ -import JSZip from 'jszip' -import { useSocket } from "@/context/SocketContext" +// React component for download button import { Button } from "@/components/ui/button" +import { useSocket } from "@/context/SocketContext" import { Download } from "lucide-react" export default function DownloadButton({ name }: { name: string }) { const { socket } = useSocket() const handleDownload = async () => { - socket?.emit("downloadFiles", {}, async (response: {files: {path: string, content: string}[]}) => { - const zip = new JSZip() - - response.files.forEach(file => { - zip.file(file.path, file.content) - }) - - const blob = await zip.generateAsync({type: "blob"}) - const url = window.URL.createObjectURL(blob) - const a = document.createElement('a') - a.href = url - a.download = `${name}.zip` - a.click() - window.URL.revokeObjectURL(url) - }) - } + socket?.emit( + "downloadFiles", + { timestamp: Date.now() }, + async (response: { zipBlob: string }) => { + const { zipBlob } = response + // Decode Base64 back to binary data + const binary = atob(zipBlob) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i) + } + const blob = new Blob([bytes], { type: "application/zip" }) + + // Create URL and download + const url = window.URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + a.download = `${name}.zip` + a.click() + window.URL.revokeObjectURL(url) + } + ) + } return (