Merge branch 'refs/heads/refactor-server'

# Conflicts:
#	backend/server/src/index.ts

.prettierignore

@@ -0,0 +1,4 @@
+frontend/**
+backend/ai/**
+backend/database/**
+backend/storage/**

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+  "editor.formatOnSave": true,
+  "editor.formatOnSaveMode": "file",
+  "editor.codeActionsOnSave": {
+    "source.fixAll": "explicit",
+    "source.organizeImports": "explicit"
+  }
+}

backend/server/.prettierrc

@@ -0,0 +1,6 @@
+{
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": false,
+    "insertFinalNewline": true
+}

backend/server/nodemon.json

@@ -1,5 +1,7 @@
 {
-  "watch": ["src"],
+  "watch": [
+    "src"
+  ],
   "ext": "ts",
   "exec": "concurrently \"npx tsc --watch\" \"ts-node src/index.ts\""
 }

backend/server/package.json

@@ -31,4 +31,4 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.4.5"
   }
-}
+}

backend/server/src/AIWorker.ts

@@ -0,0 +1,90 @@
+// AIWorker class for handling AI-related operations
+export class AIWorker {
+  private aiWorkerUrl: string
+  private cfAiKey: string
+  private databaseWorkerUrl: string
+  private workersKey: string
+
+  // Constructor to initialize AIWorker with necessary URLs and keys
+  constructor(
+    aiWorkerUrl: string,
+    cfAiKey: string,
+    databaseWorkerUrl: string,
+    workersKey: string
+  ) {
+    this.aiWorkerUrl = aiWorkerUrl
+    this.cfAiKey = cfAiKey
+    this.databaseWorkerUrl = databaseWorkerUrl
+    this.workersKey = workersKey
+  }
+
+  // Method to generate code based on user input
+  async generateCode(
+    userId: string,
+    fileName: string,
+    code: string,
+    line: number,
+    instructions: string
+  ): Promise<{ response: string; success: boolean }> {
+    try {
+      const fetchPromise = fetch(
+        `${process.env.DATABASE_WORKER_URL}/api/sandbox/generate`,
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `${process.env.WORKERS_KEY}`,
+          },
+          body: JSON.stringify({
+            userId: userId,
+          }),
+        }
+      )
+
+      // Generate code from cloudflare workers AI
+      const generateCodePromise = fetch(
+        `${process.env.AI_WORKER_URL}/api?fileName=${encodeURIComponent(
+          fileName
+        )}&code=${encodeURIComponent(code)}&line=${encodeURIComponent(
+          line
+        )}&instructions=${encodeURIComponent(instructions)}`,
+        {
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `${process.env.CF_AI_KEY}`,
+          },
+        }
+      )
+
+      const [fetchResponse, generateCodeResponse] = await Promise.all([
+        fetchPromise,
+        generateCodePromise,
+      ])
+
+      if (!generateCodeResponse.ok) {
+        throw new Error(`HTTP error! status: ${generateCodeResponse.status}`)
+      }
+
+      const reader = generateCodeResponse.body?.getReader()
+      const decoder = new TextDecoder()
+      let result = ""
+
+      if (reader) {
+        while (true) {
+          const { done, value } = await reader.read()
+          if (done) break
+          result += decoder.decode(value, { stream: true })
+        }
+      }
+
+      // The result should now contain only the modified code
+      return { response: result.trim(), success: true }
+    } catch (e: any) {
+      console.error("Error generating code:", e)
+      return {
+        response: "Error generating code. Please try again.",
+        success: false,
+      }
+    }
+  }
+}

backend/server/src/DokkuClient.ts

@@ -1,37 +1,40 @@
-import { SSHSocketClient, SSHConfig } from "./SSHSocketClient"
+import { SSHConfig, SSHSocketClient } from "./SSHSocketClient"
 
+// Interface for the response structure from Dokku commands
 export interface DokkuResponse {
-  ok: boolean;
+  ok: boolean
-  output: string;
+  output: string
 }
 
+// DokkuClient class extends SSHSocketClient to interact with Dokku via SSH
 export class DokkuClient extends SSHSocketClient {
-
   constructor(config: SSHConfig) {
-    super(
+    // Initialize with Dokku daemon socket path
-      config,
+    super(config, "/var/run/dokku-daemon/dokku-daemon.sock")
-      "/var/run/dokku-daemon/dokku-daemon.sock"
-    )
   }
 
+  // Send a command to Dokku and parse the response
   async sendCommand(command: string): Promise<DokkuResponse> {
     try {
-      const response = await this.sendData(command);
+      const response = await this.sendData(command)
 
       if (typeof response !== "string") {
-        throw new Error("Received data is not a string");
+        throw new Error("Received data is not a string")
       }
 
-      return JSON.parse(response);
+      // Parse the JSON response from Dokku
+      return JSON.parse(response)
     } catch (error: any) {
-      throw new Error(`Failed to send command: ${error.message}`);
+      throw new Error(`Failed to send command: ${error.message}`)
     }
   }
 
+  // List all deployed Dokku apps
   async listApps(): Promise<string[]> {
-    const response = await this.sendCommand("apps:list");
+    const response = await this.sendCommand("apps:list")
-    return response.output.split("\n").slice(1); // Split by newline and ignore the first line (header)
+    // Split the output by newline and remove the header
+    return response.output.split("\n").slice(1)
   }
 }
 
-export { SSHConfig };
+export { SSHConfig }

backend/server/src/FileManager.ts

@@ -0,0 +1,508 @@
+import { FilesystemEvent, Sandbox, WatchHandle } from "e2b"
+import path from "path"
+import RemoteFileStorage from "./RemoteFileStorage"
+import { MAX_BODY_SIZE } from "./ratelimit"
+import { TFile, TFileData, TFolder } from "./types"
+
+// Define the structure for sandbox files
+export type SandboxFiles = {
+  files: (TFolder | TFile)[]
+  fileData: TFileData[]
+}
+
+// Convert list of paths to the hierchical file structure used by the editor
+function generateFileStructure(paths: string[]): (TFolder | TFile)[] {
+  const root: TFolder = { id: "/", type: "folder", name: "/", children: [] }
+
+  paths.forEach((path) => {
+    const parts = path.split("/")
+    let current: TFolder = root
+
+    for (let i = 0; i < parts.length; i++) {
+      const part = parts[i]
+      const isFile = i === parts.length - 1 && part.length
+      const existing = current.children.find((child) => child.name === part)
+
+      if (existing) {
+        if (!isFile) {
+          current = existing as TFolder
+        }
+      } else {
+        if (isFile) {
+          const file: TFile = { id: `/${parts.join("/")}`, type: "file", name: part }
+          current.children.push(file)
+        } else {
+          const folder: TFolder = {
+            id: `/${parts.slice(0, i + 1).join("/")}`,
+            type: "folder",
+            name: part,
+            children: [],
+          }
+          current.children.push(folder)
+          current = folder
+        }
+      }
+    }
+  })
+
+  return root.children
+}
+
+// FileManager class to handle file operations in a sandbox
+export class FileManager {
+  private sandboxId: string
+  private sandbox: Sandbox
+  public sandboxFiles: SandboxFiles
+  private fileWatchers: WatchHandle[] = []
+  private dirName = "/home/user/project"
+  private refreshFileList: (files: SandboxFiles) => void
+
+  // Constructor to initialize the FileManager
+  constructor(
+    sandboxId: string,
+    sandbox: Sandbox,
+    refreshFileList: (files: SandboxFiles) => void
+  ) {
+    this.sandboxId = sandboxId
+    this.sandbox = sandbox
+    this.sandboxFiles = { files: [], fileData: [] }
+    this.refreshFileList = refreshFileList
+  }
+
+  // Fetch file data from list of paths
+  private async generateFileData(paths: string[]): Promise<TFileData[]> {
+    const fileData: TFileData[] = []
+
+    for (const path of paths) {
+      const parts = path.split("/")
+      const isFile = parts.length > 0 && parts[parts.length - 1].length > 0
+
+      if (isFile) {
+        const fileId = `/${parts.join("/")}`
+        const data = await RemoteFileStorage.fetchFileContent(`projects/${this.sandboxId}${fileId}`)
+        fileData.push({ id: fileId, data })
+      }
+    }
+
+    return fileData
+  }
+
+  // Convert local file path to remote path
+  private getRemoteFileId(localId: string): string {
+    return `projects/${this.sandboxId}${localId}`
+  }
+
+  // Convert remote file path to local file path
+  private getLocalFileId(remoteId: string): string | undefined {
+    const allParts = remoteId.split("/")
+    if (allParts[1] !== this.sandboxId) return undefined;
+    return allParts.slice(2).join("/")
+  }
+
+  // Convert remote file paths to local file paths
+  private getLocalFileIds(remoteIds: string[]): string[] {
+    return remoteIds
+      .map(this.getLocalFileId.bind(this))
+      .filter((id) => id !== undefined);
+  }
+
+  // Download files from remote storage
+  private async updateFileData(): Promise<TFileData[]> {
+    const remotePaths = await RemoteFileStorage.getSandboxPaths(this.sandboxId)
+    const localPaths = this.getLocalFileIds(remotePaths)
+    this.sandboxFiles.fileData = await this.generateFileData(localPaths)
+    return this.sandboxFiles.fileData
+  }
+
+  // Update file structure
+  private async updateFileStructure(): Promise<(TFolder | TFile)[]> {
+    const remotePaths = await RemoteFileStorage.getSandboxPaths(this.sandboxId)
+    const localPaths = this.getLocalFileIds(remotePaths)
+    this.sandboxFiles.files = generateFileStructure(localPaths)
+    return this.sandboxFiles.files
+  }
+
+  // Initialize the FileManager
+  async initialize() {
+
+    // Download files from remote file storage
+    await this.updateFileStructure()
+    await this.updateFileData()
+
+    // Copy all files from the project to the container
+    const promises = this.sandboxFiles.fileData.map(async (file) => {
+      try {
+        const filePath = path.join(this.dirName, file.id)
+        const parentDirectory = path.dirname(filePath)
+        if (!this.sandbox.files.exists(parentDirectory)) {
+          await this.sandbox.files.makeDir(parentDirectory)
+        }
+        await this.sandbox.files.write(filePath, file.data)
+      } catch (e: any) {
+        console.log("Failed to create file: " + e)
+      }
+    })
+    await Promise.all(promises)
+
+    // Make the logged in user the owner of all project files
+    this.fixPermissions()
+
+    await this.watchDirectory(this.dirName)
+    await this.watchSubdirectories(this.dirName)
+  }
+
+  // Check if the given path is a directory
+  private async isDirectory(directoryPath: string): Promise<boolean> {
+    try {
+      const result = await this.sandbox.commands.run(
+        `[ -d "${directoryPath}" ] && echo "true" || echo "false"`
+      )
+      return result.stdout.trim() === "true"
+    } catch (e: any) {
+      console.log("Failed to check if directory: " + e)
+      return false
+    }
+  }
+
+  // Change the owner of the project directory to user
+  private async fixPermissions() {
+    try {
+      await this.sandbox.commands.run(
+        `sudo chown -R user "${this.dirName}"`
+      )
+    } catch (e: any) {
+      console.log("Failed to fix permissions: " + e)
+    }
+  }
+
+  // Watch a directory for changes
+  async watchDirectory(directory: string): Promise<WatchHandle | undefined> {
+    try {
+      const handle = await this.sandbox.files.watch(
+        directory,
+        async (event: FilesystemEvent) => {
+          try {
+            function removeDirName(path: string, dirName: string) {
+              return path.startsWith(dirName)
+                ? path.slice(dirName.length)
+                : path
+            }
+
+            // This is the absolute file path in the container
+            const containerFilePath = path.posix.join(directory, event.name)
+            // This is the file path relative to the project directory
+            const sandboxFilePath = removeDirName(containerFilePath, this.dirName)
+            // This is the directory being watched relative to the project directory
+            const sandboxDirectory = removeDirName(directory, this.dirName)
+
+            // Helper function to find a folder by id
+            function findFolderById(
+              files: (TFolder | TFile)[],
+              folderId: string
+            ) {
+              return files.find(
+                (file: TFolder | TFile) =>
+                  file.type === "folder" && file.id === folderId
+              )
+            }
+
+            // Handle file/directory creation event
+            if (event.type === "create") {
+              const folder = findFolderById(
+                this.sandboxFiles.files,
+                sandboxDirectory
+              ) as TFolder
+              const isDir = await this.isDirectory(containerFilePath)
+
+              const newItem = isDir
+                ? ({
+                  id: sandboxFilePath,
+                  name: event.name,
+                  type: "folder",
+                  children: [],
+                } as TFolder)
+                : ({
+                  id: sandboxFilePath,
+                  name: event.name,
+                  type: "file",
+                } as TFile)
+
+              if (folder) {
+                // If the folder exists, add the new item (file/folder) as a child
+                folder.children.push(newItem)
+              } else {
+                // If folder doesn't exist, add the new item to the root
+                this.sandboxFiles.files.push(newItem)
+              }
+
+              if (!isDir) {
+                const fileData = await this.sandbox.files.read(
+                  containerFilePath
+                )
+                const fileContents =
+                  typeof fileData === "string" ? fileData : ""
+                this.sandboxFiles.fileData.push({
+                  id: sandboxFilePath,
+                  data: fileContents,
+                })
+              }
+
+              console.log(`Create ${sandboxFilePath}`)
+            }
+
+            // Handle file/directory removal or rename event
+            else if (event.type === "remove" || event.type == "rename") {
+              const folder = findFolderById(
+                this.sandboxFiles.files,
+                sandboxDirectory
+              ) as TFolder
+              const isDir = await this.isDirectory(containerFilePath)
+
+              const isFileMatch = (file: TFolder | TFile | TFileData) =>
+                file.id === sandboxFilePath ||
+                file.id.startsWith(containerFilePath + "/")
+
+              if (folder) {
+                // Remove item from its parent folder
+                folder.children = folder.children.filter(
+                  (file: TFolder | TFile) => !isFileMatch(file)
+                )
+              } else {
+                // Remove from the root if it's not inside a folder
+                this.sandboxFiles.files = this.sandboxFiles.files.filter(
+                  (file: TFolder | TFile) => !isFileMatch(file)
+                )
+              }
+
+              // Also remove any corresponding file data
+              this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter(
+                (file: TFileData) => !isFileMatch(file)
+              )
+
+              console.log(`Removed: ${sandboxFilePath}`)
+            }
+
+            // Handle file write event
+            else if (event.type === "write") {
+              const folder = findFolderById(
+                this.sandboxFiles.files,
+                sandboxDirectory
+              ) as TFolder
+              const fileToWrite = this.sandboxFiles.fileData.find(
+                (file) => file.id === sandboxFilePath
+              )
+
+              if (fileToWrite) {
+                fileToWrite.data = await this.sandbox.files.read(
+                  containerFilePath
+                )
+                console.log(`Write to ${sandboxFilePath}`)
+              } else {
+                // If the file is part of a folder structure, locate it and update its data
+                const fileInFolder = folder?.children.find(
+                  (file) => file.id === sandboxFilePath
+                )
+                if (fileInFolder) {
+                  const fileData = await this.sandbox.files.read(
+                    containerFilePath
+                  )
+                  const fileContents =
+                    typeof fileData === "string" ? fileData : ""
+                  this.sandboxFiles.fileData.push({
+                    id: sandboxFilePath,
+                    data: fileContents,
+                  })
+                  console.log(`Write to ${sandboxFilePath}`)
+                }
+              }
+            }
+
+            // Tell the client to reload the file list
+            this.refreshFileList(this.sandboxFiles)
+          } catch (error) {
+            console.error(
+              `Error handling ${event.type} event for ${event.name}:`,
+              error
+            )
+          }
+        },
+        { timeout: 0 }
+      )
+      this.fileWatchers.push(handle)
+      return handle
+    } catch (error) {
+      console.error(`Error watching filesystem:`, error)
+    }
+  }
+
+  // Watch subdirectories recursively
+  async watchSubdirectories(directory: string) {
+    const dirContent = await this.sandbox.files.list(directory)
+    await Promise.all(
+      dirContent.map(async (item) => {
+        if (item.type === "dir") {
+          console.log("Watching " + item.path)
+          await this.watchDirectory(item.path)
+        }
+      })
+    )
+  }
+
+  // Get file content
+  async getFile(fileId: string): Promise<string | undefined> {
+    const file = this.sandboxFiles.fileData.find((f) => f.id === fileId)
+    return file?.data
+  }
+
+  // Get folder content
+  async getFolder(folderId: string): Promise<string[]> {
+    const remotePaths = await RemoteFileStorage.getFolder(this.getRemoteFileId(folderId))
+    return this.getLocalFileIds(remotePaths)
+  }
+
+  // Save file content
+  async saveFile(fileId: string, body: string): Promise<void> {
+    if (!fileId) return // handles saving when no file is open
+
+    if (Buffer.byteLength(body, "utf-8") > MAX_BODY_SIZE) {
+      throw new Error("File size too large. Please reduce the file size.")
+    }
+    await RemoteFileStorage.saveFile(this.getRemoteFileId(fileId), body)
+    const file = this.sandboxFiles.fileData.find((f) => f.id === fileId)
+    if (!file) return
+    file.data = body
+
+    await this.sandbox.files.write(path.posix.join(this.dirName, file.id), body)
+    this.fixPermissions()
+  }
+
+  // Move a file to a different folder
+  async moveFile(
+    fileId: string,
+    folderId: string
+  ): Promise<(TFolder | TFile)[]> {
+    const fileData = this.sandboxFiles.fileData.find((f) => f.id === fileId)
+    const file = this.sandboxFiles.files.find((f) => f.id === fileId)
+    if (!fileData || !file) return this.sandboxFiles.files
+
+    const parts = fileId.split("/")
+    const newFileId = folderId + "/" + parts.pop()
+
+    await this.moveFileInContainer(fileId, newFileId)
+
+    await this.fixPermissions()
+
+    fileData.id = newFileId
+    file.id = newFileId
+
+    await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data)
+    return this.updateFileStructure()
+  }
+
+  // Move a file within the container
+  private async moveFileInContainer(oldPath: string, newPath: string) {
+    try {
+      const fileContents = await this.sandbox.files.read(
+        path.posix.join(this.dirName, oldPath)
+      )
+      await this.sandbox.files.write(
+        path.posix.join(this.dirName, newPath),
+        fileContents
+      )
+      await this.sandbox.files.remove(path.posix.join(this.dirName, oldPath))
+    } catch (e) {
+      console.error(`Error moving file from ${oldPath} to ${newPath}:`, e)
+    }
+  }
+
+  // Create a new file
+  async createFile(name: string): Promise<boolean> {
+    const size: number = await RemoteFileStorage.getProjectSize(this.sandboxId)
+    if (size > 200 * 1024 * 1024) {
+      throw new Error("Project size exceeded. Please delete some files.")
+    }
+
+    const id = `/${name}`
+
+    await this.sandbox.files.write(path.posix.join(this.dirName, id), "")
+    await this.fixPermissions()
+
+    this.sandboxFiles.files.push({
+      id,
+      name,
+      type: "file",
+    })
+
+    this.sandboxFiles.fileData.push({
+      id,
+      data: "",
+    })
+
+    await RemoteFileStorage.createFile(this.getRemoteFileId(id))
+
+    return true
+  }
+
+  // Create a new folder
+  async createFolder(name: string): Promise<void> {
+    const id = `/${name}`
+    await this.sandbox.files.makeDir(path.posix.join(this.dirName, id))
+  }
+
+  // Rename a file
+  async renameFile(fileId: string, newName: string): Promise<void> {
+    const fileData = this.sandboxFiles.fileData.find((f) => f.id === fileId)
+    const file = this.sandboxFiles.files.find((f) => f.id === fileId)
+    if (!fileData || !file) return
+
+    const parts = fileId.split("/")
+    const newFileId = parts.slice(0, parts.length - 1).join("/") + "/" + newName
+
+    await this.moveFileInContainer(fileId, newFileId)
+    await this.fixPermissions()
+    await RemoteFileStorage.renameFile(this.getRemoteFileId(fileId), this.getRemoteFileId(newFileId), fileData.data)
+
+    fileData.id = newFileId
+    file.id = newFileId
+  }
+
+  // Delete a file
+  async deleteFile(fileId: string): Promise<(TFolder | TFile)[]> {
+    const file = this.sandboxFiles.fileData.find((f) => f.id === fileId)
+    if (!file) return this.sandboxFiles.files
+
+    await this.sandbox.files.remove(path.posix.join(this.dirName, fileId))
+    this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter(
+      (f) => f.id !== fileId
+    )
+
+    await RemoteFileStorage.deleteFile(this.getRemoteFileId(fileId))
+    return this.updateFileStructure()
+  }
+
+  // Delete a folder
+  async deleteFolder(folderId: string): Promise<(TFolder | TFile)[]> {
+    const files = await RemoteFileStorage.getFolder(this.getRemoteFileId(folderId))
+
+    await Promise.all(
+      files.map(async (file) => {
+        await this.sandbox.files.remove(path.posix.join(this.dirName, file))
+        this.sandboxFiles.fileData = this.sandboxFiles.fileData.filter(
+          (f) => f.id !== file
+        )
+        await RemoteFileStorage.deleteFile(this.getRemoteFileId(file))
+      })
+    )
+
+    return this.updateFileStructure()
+  }
+
+  // Close all file watchers
+  async closeWatchers() {
+    await Promise.all(
+      this.fileWatchers.map(async (handle: WatchHandle) => {
+        await handle.close()
+      })
+    )
+  }
+}

backend/server/src/RemoteFileStorage.ts

@@ -0,0 +1,117 @@
+import * as dotenv from "dotenv"
+import { R2Files } from "./types"
+
+dotenv.config()
+
+export const RemoteFileStorage = {
+  getSandboxPaths: async (id: string) => {
+    const res = await fetch(
+      `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`,
+      {
+        headers: {
+          Authorization: `${process.env.WORKERS_KEY}`,
+        },
+      }
+    )
+    const data: R2Files = await res.json()
+
+    return data.objects.map((obj) => obj.key)
+  },
+
+  getFolder: async (folderId: string) => {
+    const res = await fetch(
+      `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`,
+      {
+        headers: {
+          Authorization: `${process.env.WORKERS_KEY}`,
+        },
+      }
+    )
+    const data: R2Files = await res.json()
+
+    return data.objects.map((obj) => obj.key)
+  },
+
+  fetchFileContent: async (fileId: string): Promise<string> => {
+    try {
+      const fileRes = await fetch(
+        `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`,
+        {
+          headers: {
+            Authorization: `${process.env.WORKERS_KEY}`,
+          },
+        }
+      )
+      return await fileRes.text()
+    } catch (error) {
+      console.error("ERROR fetching file:", error)
+      return ""
+    }
+  },
+
+  createFile: async (fileId: string) => {
+    const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `${process.env.WORKERS_KEY}`,
+      },
+      body: JSON.stringify({ fileId }),
+    })
+    return res.ok
+  },
+
+  renameFile: async (
+    fileId: string,
+    newFileId: string,
+    data: string
+  ) => {
+    const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `${process.env.WORKERS_KEY}`,
+      },
+      body: JSON.stringify({ fileId, newFileId, data }),
+    })
+    return res.ok
+  },
+
+  saveFile: async (fileId: string, data: string) => {
+    const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `${process.env.WORKERS_KEY}`,
+      },
+      body: JSON.stringify({ fileId, data }),
+    })
+    return res.ok
+  },
+
+  deleteFile: async (fileId: string) => {
+    const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, {
+      method: "DELETE",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `${process.env.WORKERS_KEY}`,
+      },
+      body: JSON.stringify({ fileId }),
+    })
+    return res.ok
+  },
+
+  getProjectSize: async (id: string) => {
+    const res = await fetch(
+      `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`,
+      {
+        headers: {
+          Authorization: `${process.env.WORKERS_KEY}`,
+        },
+      }
+    )
+    return (await res.json()).size
+  }
+}
+
+export default RemoteFileStorage

backend/server/src/SSHSocketClient.ts

@@ -1,90 +1,98 @@
-import { Client } from "ssh2";
+import { Client } from "ssh2"
 
+// Interface defining the configuration for SSH connection
 export interface SSHConfig {
-  host: string;
+  host: string
-  port?: number;
+  port?: number
-  username: string;
+  username: string
-  privateKey: Buffer;
+  privateKey: Buffer
 }
 
+// Class to handle SSH connections and communicate with a Unix socket
 export class SSHSocketClient {
-    private conn: Client;
+  private conn: Client
-    private config: SSHConfig;
+  private config: SSHConfig
-    private socketPath: string;
+  private socketPath: string
-    private isConnected: boolean = false;
+  private isConnected: boolean = false
-  
+
-    constructor(config: SSHConfig, socketPath: string) {
+  // Constructor initializes the SSH client and sets up configuration
-      this.conn = new Client();
+  constructor(config: SSHConfig, socketPath: string) {
-      this.config = { ...config, port: 22};
+    this.conn = new Client()
-      this.socketPath = socketPath;
+    this.config = { ...config, port: 22 } // Default port to 22 if not provided
-  
+    this.socketPath = socketPath
-      this.setupTerminationHandlers();
+
-    }
+    this.setupTerminationHandlers()
-  
+  }
-    private setupTerminationHandlers() {
+
-      process.on("SIGINT", this.closeConnection.bind(this));
+  // Set up handlers for graceful termination
-      process.on("SIGTERM", this.closeConnection.bind(this));
+  private setupTerminationHandlers() {
-    }
+    process.on("SIGINT", this.closeConnection.bind(this))
-  
+    process.on("SIGTERM", this.closeConnection.bind(this))
-    private closeConnection() {
+  }
-      console.log("Closing SSH connection...");
+
-      this.conn.end();
+  // Method to close the SSH connection
-      this.isConnected = false;
+  private closeConnection() {
-      process.exit(0);
+    console.log("Closing SSH connection...")
-    }
+    this.conn.end()
-  
+    this.isConnected = false
-    connect(): Promise<void> {
+    process.exit(0)
-      return new Promise((resolve, reject) => {
+  }
-        this.conn
+
-          .on("ready", () => {
+  // Method to establish the SSH connection
-            console.log("SSH connection established");
+  connect(): Promise<void> {
-            this.isConnected = true;
+    return new Promise((resolve, reject) => {
-            resolve();
+      this.conn
-          })
+        .on("ready", () => {
-          .on("error", (err) => {
+          console.log("SSH connection established")
-            console.error("SSH connection error:", err);
+          this.isConnected = true
-            this.isConnected = false;
+          resolve()
-            reject(err);
+        })
-          })
+        .on("error", (err) => {
-          .on("close", () => {
+          console.error("SSH connection error:", err)
-            console.log("SSH connection closed");
+          this.isConnected = false
-            this.isConnected = false;
+          reject(err)
-          })
+        })
-          .connect(this.config);
+        .on("close", () => {
-      });
+          console.log("SSH connection closed")
-    }
+          this.isConnected = false
-  
+        })
-    sendData(data: string): Promise<string> {
+        .connect(this.config)
-      return new Promise((resolve, reject) => {
+    })
-        if (!this.isConnected) {
+  }
-          reject(new Error("SSH connection is not established"));
+
-          return;
+  // Method to send data through the SSH connection to the Unix socket
-        }
+  sendData(data: string): Promise<string> {
-  
+    return new Promise((resolve, reject) => {
-        this.conn.exec(
+      if (!this.isConnected) {
-          `echo "${data}" | nc -U ${this.socketPath}`,
+        reject(new Error("SSH connection is not established"))
-          (err, stream) => {
+        return
-            if (err) {
+      }
-              reject(err);
+
-              return;
+      // Use netcat to send data to the Unix socket
-            }
+      this.conn.exec(
-  
+        `echo "${data}" | nc -U ${this.socketPath}`,
-            stream
+        (err, stream) => {
-              .on("close", (code: number, signal: string) => {
+          if (err) {
-                reject(
+            reject(err)
-                  new Error(
+            return
-                    `Stream closed with code ${code} and signal ${signal}`
-                  )
-                );
-              })
-              .on("data", (data: Buffer) => {
-                resolve(data.toString());
-              })
-              .stderr.on("data", (data: Buffer) => {
-                reject(new Error(data.toString()));
-              });
           }
-        );
+
-      });
+          stream
-    }
+            .on("close", (code: number, signal: string) => {
-  }
+              reject(
+                new Error(
+                  `Stream closed with code ${code} and signal ${signal}`
+                )
+              )
+            })
+            .on("data", (data: Buffer) => {
+              resolve(data.toString())
+            })
+            .stderr.on("data", (data: Buffer) => {
+              reject(new Error(data.toString()))
+            })
+        }
+      )
+    })
+  }
+}

backend/server/src/SecureGitClient.ts

@@ -1,82 +1,84 @@
-import simpleGit, { SimpleGit } from "simple-git";
+import fs from "fs"
-import path from "path";
+import os from "os"
-import fs from "fs";
+import path from "path"
-import os from "os";
+import simpleGit, { SimpleGit } from "simple-git"
 
 export type FileData = {
-  id: string;
+  id: string
-  data: string;
+  data: string
-};
+}
 
 export class SecureGitClient {
-  private gitUrl: string;
+  private gitUrl: string
-  private sshKeyPath: string;
+  private sshKeyPath: string
 
   constructor(gitUrl: string, sshKeyPath: string) {
-    this.gitUrl = gitUrl;
+    this.gitUrl = gitUrl
-    this.sshKeyPath = sshKeyPath;
+    this.sshKeyPath = sshKeyPath
   }
 
   async pushFiles(fileData: FileData[], repository: string): Promise<void> {
-    let tempDir: string | undefined;
+    let tempDir: string | undefined
 
     try {
       // Create a temporary directory
-      tempDir = fs.mkdtempSync(path.posix.join(os.tmpdir(), 'git-push-'));
+      tempDir = fs.mkdtempSync(path.posix.join(os.tmpdir(), "git-push-"))
-      console.log(`Temporary directory created: ${tempDir}`);
+      console.log(`Temporary directory created: ${tempDir}`)
 
       // Write files to the temporary directory
-      console.log(`Writing ${fileData.length} files.`);
+      console.log(`Writing ${fileData.length} files.`)
       for (const { id, data } of fileData) {
-        const filePath = path.posix.join(tempDir, id);
+        const filePath = path.posix.join(tempDir, id)
-        const dirPath = path.dirname(filePath);
+        const dirPath = path.dirname(filePath)
-      
+
         if (!fs.existsSync(dirPath)) {
-          fs.mkdirSync(dirPath, { recursive: true });
+          fs.mkdirSync(dirPath, { recursive: true })
         }
-        fs.writeFileSync(filePath, data);
+        fs.writeFileSync(filePath, data)
       }
 
       // Initialize the simple-git instance with the temporary directory and custom SSH command
       const git: SimpleGit = simpleGit(tempDir, {
         config: [
-          'core.sshCommand=ssh -i ' + this.sshKeyPath + ' -o IdentitiesOnly=yes'
+          "core.sshCommand=ssh -i " +
-        ]
+            this.sshKeyPath +
+            " -o IdentitiesOnly=yes",
+        ],
       }).outputHandler((_command, stdout, stderr) => {
-        stdout.pipe(process.stdout);
+        stdout.pipe(process.stdout)
-        stderr.pipe(process.stderr);
+        stderr.pipe(process.stderr)
-     });;
+      })
 
       // Initialize a new Git repository
-      await git.init();
+      await git.init()
 
       // Add remote repository
-      await git.addRemote("origin", `${this.gitUrl}:${repository}`);
+      await git.addRemote("origin", `${this.gitUrl}:${repository}`)
 
       // Add files to the repository
-      for (const {id, data} of fileData) {
+      for (const { id, data } of fileData) {
-        await git.add(id);
+        await git.add(id)
       }
 
       // Commit the changes
-      await git.commit("Add files.");
+      await git.commit("Add files.")
 
       // Push the changes to the remote repository
-      await git.push("origin", "master", {'--force': null});
+      await git.push("origin", "master", { "--force": null })
 
-      console.log("Files successfully pushed to the repository");
+      console.log("Files successfully pushed to the repository")
 
       if (tempDir) {
-        fs.rmSync(tempDir, { recursive: true, force: true });
+        fs.rmSync(tempDir, { recursive: true, force: true })
-        console.log(`Temporary directory removed: ${tempDir}`);
+        console.log(`Temporary directory removed: ${tempDir}`)
       }
     } catch (error) {
       if (tempDir) {
-        fs.rmSync(tempDir, { recursive: true, force: true });
+        fs.rmSync(tempDir, { recursive: true, force: true })
-        console.log(`Temporary directory removed: ${tempDir}`);
+        console.log(`Temporary directory removed: ${tempDir}`)
       }
-      console.error("Error pushing files to the repository:", error);
+      console.error("Error pushing files to the repository:", error)
-      throw error;
+      throw error
     }
   }
-}
+}

backend/server/src/Terminal.ts

@@ -1,13 +1,13 @@
-import { Sandbox, ProcessHandle } from "e2b";
+import { ProcessHandle, Sandbox } from "e2b"
 
 // Terminal class to manage a pseudo-terminal (PTY) in a sandbox environment
 export class Terminal {
-  private pty: ProcessHandle | undefined; // Holds the PTY process handle
+  private pty: ProcessHandle | undefined // Holds the PTY process handle
-  private sandbox: Sandbox; // Reference to the sandbox environment
+  private sandbox: Sandbox // Reference to the sandbox environment
 
   // Constructor initializes the Terminal with a sandbox
   constructor(sandbox: Sandbox) {
-    this.sandbox = sandbox;
+    this.sandbox = sandbox
   }
 
   // Initialize the terminal with specified rows, columns, and data handler
@@ -16,9 +16,9 @@ export class Terminal {
     cols = 80,
     onData,
   }: {
-    rows?: number;
+    rows?: number
-    cols?: number;
+    cols?: number
-    onData: (responseData: string) => void;
+    onData: (responseData: string) => void
   }): Promise<void> {
     // Create a new PTY process
     this.pty = await this.sandbox.pty.create({
@@ -26,35 +26,38 @@ export class Terminal {
       cols,
       timeout: 0,
       onData: (data: Uint8Array) => {
-        onData(new TextDecoder().decode(data)); // Convert received data to string and pass to handler
+        onData(new TextDecoder().decode(data)) // Convert received data to string and pass to handler
       },
-    });
+    })
   }
 
   // Send data to the terminal
   async sendData(data: string) {
     if (this.pty) {
-      await this.sandbox.pty.sendInput(this.pty.pid, new TextEncoder().encode(data));
+      await this.sandbox.pty.sendInput(
+        this.pty.pid,
+        new TextEncoder().encode(data)
+      )
     } else {
-      console.log("Cannot send data because pty is not initialized.");
+      console.log("Cannot send data because pty is not initialized.")
     }
   }
 
   // Resize the terminal
   async resize(size: { cols: number; rows: number }): Promise<void> {
     if (this.pty) {
-      await this.sandbox.pty.resize(this.pty.pid, size);
+      await this.sandbox.pty.resize(this.pty.pid, size)
     } else {
-      console.log("Cannot resize terminal because pty is not initialized.");
+      console.log("Cannot resize terminal because pty is not initialized.")
     }
   }
 
   // Close the terminal, killing the PTY process and stopping the input stream
   async close(): Promise<void> {
     if (this.pty) {
-      await this.pty.kill();
+      await this.pty.kill()
     } else {
-      console.log("Cannot kill pty because it is not initialized.");
+      console.log("Cannot kill pty because it is not initialized.")
     }
   }
 }
@@ -64,4 +67,4 @@ export class Terminal {
 // await terminal.init();
 // terminal.sendData('ls -la');
 // await terminal.resize({ cols: 100, rows: 30 });
-// await terminal.close();
+// await terminal.close();

backend/server/src/TerminalManager.ts

@@ -0,0 +1,74 @@
+import { Sandbox } from "e2b"
+import { Terminal } from "./Terminal"
+
+export class TerminalManager {
+  private sandbox: Sandbox
+  private terminals: Record<string, Terminal> = {}
+
+  constructor(sandbox: Sandbox) {
+    this.sandbox = sandbox
+  }
+
+  async createTerminal(
+    id: string,
+    onData: (responseString: string) => void
+  ): Promise<void> {
+    if (this.terminals[id]) {
+      return
+    }
+
+    this.terminals[id] = new Terminal(this.sandbox)
+    await this.terminals[id].init({
+      onData,
+      cols: 80,
+      rows: 20,
+    })
+
+    const defaultDirectory = "/home/user/project"
+    const defaultCommands = [
+      `cd "${defaultDirectory}"`,
+      "export PS1='user> '",
+      "clear",
+    ]
+    for (const command of defaultCommands) {
+      await this.terminals[id].sendData(command + "\r")
+    }
+
+    console.log("Created terminal", id)
+  }
+
+  async resizeTerminal(dimensions: {
+    cols: number
+    rows: number
+  }): Promise<void> {
+    Object.values(this.terminals).forEach((t) => {
+      t.resize(dimensions)
+    })
+  }
+
+  async sendTerminalData(id: string, data: string): Promise<void> {
+    if (!this.terminals[id]) {
+      return
+    }
+
+    await this.terminals[id].sendData(data)
+  }
+
+  async closeTerminal(id: string): Promise<void> {
+    if (!this.terminals[id]) {
+      return
+    }
+
+    await this.terminals[id].close()
+    delete this.terminals[id]
+  }
+
+  async closeAllTerminals(): Promise<void> {
+    await Promise.all(
+      Object.entries(this.terminals).map(async ([key, terminal]) => {
+        await terminal.close()
+        delete this.terminals[key]
+      })
+    )
+  }
+}

backend/server/src/fileoperations.ts

@@ -1,177 +0,0 @@
-import * as dotenv from "dotenv";
-import {
-  R2FileBody,
-  R2Files,
-  Sandbox,
-  TFile,
-  TFileData,
-  TFolder,
-} from "./types";
-
-dotenv.config();
-
-export const getSandboxFiles = async (id: string) => {
-  const res = await fetch(
-    `${process.env.STORAGE_WORKER_URL}/api?sandboxId=${id}`,
-    {
-      headers: {
-        Authorization: `${process.env.WORKERS_KEY}`,
-      },
-    }
-  );
-  const data: R2Files = await res.json();
-
-  const paths = data.objects.map((obj) => obj.key);
-  const processedFiles = await processFiles(paths, id);
-  return processedFiles;
-};
-
-export const getFolder = async (folderId: string) => {
-  const res = await fetch(
-    `${process.env.STORAGE_WORKER_URL}/api?folderId=${folderId}`,
-    {
-      headers: {
-        Authorization: `${process.env.WORKERS_KEY}`,
-      },
-    }
-  );
-  const data: R2Files = await res.json();
-
-  return data.objects.map((obj) => obj.key);
-};
-
-const processFiles = async (paths: string[], id: string) => {
-  const root: TFolder = { id: "/", type: "folder", name: "/", children: [] };
-  const fileData: TFileData[] = [];
-
-  paths.forEach((path) => {
-    const allParts = path.split("/");
-    if (allParts[1] !== id) {
-      return;
-    }
-
-    const parts = allParts.slice(2);
-    let current: TFolder = root;
-
-    for (let i = 0; i < parts.length; i++) {
-      const part = parts[i];
-      const isFile = i === parts.length - 1 && part.length;
-      const existing = current.children.find((child) => child.name === part);
-
-      if (existing) {
-        if (!isFile) {
-          current = existing as TFolder;
-        }
-      } else {
-        if (isFile) {
-          const file: TFile = { id: path, type: "file", name: part };
-          current.children.push(file);
-          fileData.push({ id: path, data: "" });
-        } else {
-          const folder: TFolder = {
-            // id: path, // todo: wrong id. for example, folder "src" ID is: projects/a7vgttfqbgy403ratp7du3ln/src/App.css
-            id: `projects/${id}/${parts.slice(0, i + 1).join("/")}`,
-            type: "folder",
-            name: part,
-            children: [],
-          };
-          current.children.push(folder);
-          current = folder;
-        }
-      }
-    }
-  });
-
-  await Promise.all(
-    fileData.map(async (file) => {
-      const data = await fetchFileContent(file.id);
-      file.data = data;
-    })
-  );
-
-  return {
-    files: root.children,
-    fileData,
-  };
-};
-
-const fetchFileContent = async (fileId: string): Promise<string> => {
-  try {
-    const fileRes = await fetch(
-      `${process.env.STORAGE_WORKER_URL}/api?fileId=${fileId}`,
-      {
-        headers: {
-          Authorization: `${process.env.WORKERS_KEY}`,
-        },
-      }
-    );
-    return await fileRes.text();
-  } catch (error) {
-    console.error("ERROR fetching file:", error);
-    return "";
-  }
-};
-
-export const createFile = async (fileId: string) => {
-  const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `${process.env.WORKERS_KEY}`,
-    },
-    body: JSON.stringify({ fileId }),
-  });
-  return res.ok;
-};
-
-export const renameFile = async (
-  fileId: string,
-  newFileId: string,
-  data: string
-) => {
-  const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/rename`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `${process.env.WORKERS_KEY}`,
-    },
-    body: JSON.stringify({ fileId, newFileId, data }),
-  });
-  return res.ok;
-};
-
-export const saveFile = async (fileId: string, data: string) => {
-  const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api/save`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `${process.env.WORKERS_KEY}`,
-    },
-    body: JSON.stringify({ fileId, data }),
-  });
-  return res.ok;
-};
-
-export const deleteFile = async (fileId: string) => {
-  const res = await fetch(`${process.env.STORAGE_WORKER_URL}/api`, {
-    method: "DELETE",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `${process.env.WORKERS_KEY}`,
-    },
-    body: JSON.stringify({ fileId }),
-  });
-  return res.ok;
-};
-
-export const getProjectSize = async (id: string) => {
-  const res = await fetch(
-    `${process.env.STORAGE_WORKER_URL}/api/size?sandboxId=${id}`,
-    {
-      headers: {
-        Authorization: `${process.env.WORKERS_KEY}`,
-      },
-    }
-  );
-  return (await res.json()).size;
-};

backend/server/src/ratelimit.ts

@@ -30,4 +30,4 @@ export const deleteFileRL = new RateLimiterMemory({
 export const deleteFolderRL = new RateLimiterMemory({
   points: 1,
   duration: 2,
-})
+})

backend/server/src/types.ts

@@ -1,70 +1,70 @@
 // DB Types
 
 export type User = {
-  id: string;
+  id: string
-  name: string;
+  name: string
-  email: string;
+  email: string
-  generations: number;
+  generations: number
-  sandbox: Sandbox[];
+  sandbox: Sandbox[]
-  usersToSandboxes: UsersToSandboxes[];
+  usersToSandboxes: UsersToSandboxes[]
-};
+}
 
 export type Sandbox = {
-  id: string;
+  id: string
-  name: string;
+  name: string
-  type: "react" | "node";
+  type: "react" | "node"
-  visibility: "public" | "private";
+  visibility: "public" | "private"
-  createdAt: Date;
+  createdAt: Date
-  userId: string;
+  userId: string
-  usersToSandboxes: UsersToSandboxes[];
+  usersToSandboxes: UsersToSandboxes[]
-};
+}
 
 export type UsersToSandboxes = {
-  userId: string;
+  userId: string
-  sandboxId: string;
+  sandboxId: string
-  sharedOn: Date;
+  sharedOn: Date
-};
+}
 
 export type TFolder = {
-  id: string;
+  id: string
-  type: "folder";
+  type: "folder"
-  name: string;
+  name: string
-  children: (TFile | TFolder)[];
+  children: (TFile | TFolder)[]
-};
+}
 
 export type TFile = {
-  id: string;
+  id: string
-  type: "file";
+  type: "file"
-  name: string;
+  name: string
-};
+}
 
 export type TFileData = {
-  id: string;
+  id: string
-  data: string;
+  data: string
-};
+}
 
 export type R2Files = {
-  objects: R2FileData[];
+  objects: R2FileData[]
-  truncated: boolean;
+  truncated: boolean
-  delimitedPrefixes: any[];
+  delimitedPrefixes: any[]
-};
+}
 
 export type R2FileData = {
-  storageClass: string;
+  storageClass: string
-  uploaded: string;
+  uploaded: string
-  checksums: any;
+  checksums: any
-  httpEtag: string;
+  httpEtag: string
-  etag: string;
+  etag: string
-  size: number;
+  size: number
-  version: string;
+  version: string
-  key: string;
+  key: string
-};
+}
 
 export type R2FileBody = R2FileData & {
-  body: ReadableStream;
+  body: ReadableStream
-  bodyUsed: boolean;
+  bodyUsed: boolean
-  arrayBuffer: Promise<ArrayBuffer>;
+  arrayBuffer: Promise<ArrayBuffer>
-  text: Promise<string>;
+  text: Promise<string>
-  json: Promise<any>;
+  json: Promise<any>
-  blob: Promise<Blob>;
+  blob: Promise<Blob>
-};
+}

backend/server/src/utils.ts

@@ -1,23 +1,23 @@
 export class LockManager {
-  private locks: { [key: string]: Promise<any> };
+  private locks: { [key: string]: Promise<any> }
 
   constructor() {
-    this.locks = {};
+    this.locks = {}
   }
 
   async acquireLock<T>(key: string, task: () => Promise<T>): Promise<T> {
     if (!this.locks[key]) {
       this.locks[key] = new Promise<T>(async (resolve, reject) => {
         try {
-          const result = await task();
+          const result = await task()
-          resolve(result);
+          resolve(result)
         } catch (error) {
-          reject(error);
+          reject(error)
         } finally {
-          delete this.locks[key];
+          delete this.locks[key]
         }
-      });
+      })
     }
-    return await this.locks[key];
+    return await this.locks[key]
   }
-}
+}