Merge branch 'refs/heads/sync-container-files'

# Conflicts:
#	backend/server/src/index.ts

backend/server/src/index.ts

@@ -6,10 +6,15 @@ import { createServer } from "http";
 import { Server } from "socket.io";
 import { DokkuClient } from "./DokkuClient";
 import { SecureGitClient, FileData } from "./SecureGitClient";
-import fs from "fs";
+import fs, { readFile } from "fs";
 
 import { z } from "zod";
-import { User } from "./types";
+import {
+  TFile,
+  TFileData,
+  TFolder,
+  User
+} from "./types";
 import {
   createFile,
   deleteFile,
@@ -21,7 +26,7 @@ import {
 } from "./fileoperations";
 import { LockManager } from "./utils";
 
-import { Sandbox, Filesystem } from "e2b";
+import { Sandbox, Filesystem, FilesystemEvent, EntryInfo, WatchHandle } from "e2b";
 
 import { Terminal } from "./Terminal"
 
@@ -34,6 +39,18 @@ import {
   saveFileRL,
 } from "./ratelimit";
 
+process.on('uncaughtException', (error) => {
+  console.error('Uncaught Exception:', error);
+  // Do not exit the process
+  // You can add additional logging or recovery logic here
+});
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  // Do not exit the process
+  // You can also handle the rejected promise here if needed
+});
+
 // The amount of time in ms that a container will stay alive without a hearbeat.
 const CONTAINER_TIMEOUT = 60_000;
 
@@ -58,14 +75,14 @@ const terminals: Record<string, Terminal> = {};
 
 const dirName = "/home/user";
 
-const moveFile = async (
+const moveFile = async (filesystem: Filesystem, filePath: string, newFilePath: string) => {
-  filesystem: Filesystem,
+  try {
-  filePath: string,
+    const fileContents = await filesystem.read(filePath);
-  newFilePath: string
+    await filesystem.write(newFilePath, fileContents);
-) => {
+    await filesystem.remove(filePath);
-  const fileContents = await filesystem.read(filePath);
+  } catch (e) {
-  await filesystem.write(newFilePath, fileContents);
+    console.error(`Error moving file from ${filePath} to ${newFilePath}:`, e);
-  await filesystem.remove(filePath);
+  }
 };
 
 io.use(async (socket, next) => {
@@ -160,12 +177,13 @@ io.on("connection", async (socket) => {
       }
     }
 
-    await lockManager.acquireLock(data.sandboxId, async () => {
+    const createdContainer = await lockManager.acquireLock(data.sandboxId, async () => {
       try {
         // Start a new container if the container doesn't exist or it timed out.
         if (!containers[data.sandboxId] || !(await containers[data.sandboxId].isRunning())) {
           containers[data.sandboxId] = await Sandbox.create({ timeoutMs: CONTAINER_TIMEOUT });
           console.log("Created container ", data.sandboxId);
+          return true;
         }
       } catch (e: any) {
         console.error(`Error creating container ${data.sandboxId}:`, e);
@@ -173,32 +191,176 @@ io.on("connection", async (socket) => {
       }
     });
 
+    const sandboxFiles = await getSandboxFiles(data.sandboxId);
+    const projectDirectory = path.join(dirName, "projects", data.sandboxId);
+    const containerFiles = containers[data.sandboxId].files;
+    const fileWatchers: WatchHandle[] = [];
+
     // Change the owner of the project directory to user
-    const fixPermissions = async () => {
+    const fixPermissions = async (projectDirectory: string) => {
-      await containers[data.sandboxId].commands.run(
+      try {
-        `sudo chown -R user "${path.posix.join(dirName, "projects", data.sandboxId)}"`
+        await containers[data.sandboxId].commands.run(
-      );
+          `sudo chown -R user "${projectDirectory}"`
+        );
+      } catch (e: any) {
+        console.log("Failed to fix permissions: " + e);
+      }
     };
 
-    // Copy all files from the project to the container
+    // Check if the given path is a directory
-    const sandboxFiles = await getSandboxFiles(data.sandboxId);
+    const isDirectory = async (projectDirectory: string): Promise<boolean> => {
-    const containerFiles = containers[data.sandboxId].files;
-    const promises = sandboxFiles.fileData.map(async (file) => {
       try {
-        const filePath = path.posix.join(dirName, file.id);
+        const result = await containers[data.sandboxId].commands.run(
-        const parentDirectory = path.dirname(filePath);
+          `[ -d "${projectDirectory}" ] && echo "true" || echo "false"`
-        if (!containerFiles.exists(parentDirectory)) {
+        );
-          await containerFiles.makeDir(parentDirectory);
+        return result.stdout.trim() === "true";
-        }
-        await containerFiles.write(filePath, file.data);
       } catch (e: any) {
-        console.log("Failed to create file: " + e);
+        console.log("Failed to check if directory: " + e);
+        return false;
       }
-    });
+    };
-    await Promise.all(promises);
 
-    fixPermissions();
+    // Only continue to container setup if a new container was created
+    if (createdContainer) {
 
+      // Copy all files from the project to the container
+      const promises = sandboxFiles.fileData.map(async (file) => {
+        try {
+          const filePath = path.join(dirName, file.id);
+          const parentDirectory = path.dirname(filePath);
+          if (!containerFiles.exists(parentDirectory)) {
+            await containerFiles.makeDir(parentDirectory);
+          }
+          await containerFiles.write(filePath, file.data);
+        } catch (e: any) {
+          console.log("Failed to create file: " + e);
+        }
+      });
+      await Promise.all(promises);
+
+      // Make the logged in user the owner of all project files
+      fixPermissions(projectDirectory);
+
+    }
+
+    // Start filesystem watcher for the project directory
+    const watchDirectory = async (directory: string): Promise<WatchHandle | undefined> => {
+      try {
+        return await containerFiles.watch(directory, async (event: FilesystemEvent) => {
+          try {
+
+            function removeDirName(path : string, dirName : string) {
+                return path.startsWith(dirName) ? path.slice(dirName.length) : path;
+            }
+
+            // This is the absolute file path in the container
+            const containerFilePath = path.join(directory, event.name);
+            // This is the file path relative to the home directory
+            const sandboxFilePath = removeDirName(containerFilePath, dirName + "/");
+            // This is the directory being watched relative to the home directory
+            const sandboxDirectory = removeDirName(directory, dirName + "/");
+
+            // Helper function to find a folder by id
+            function findFolderById(files: (TFolder | TFile)[], folderId : string) {
+              return files.find((file : TFolder | TFile) => file.type === "folder" && file.id === folderId);
+            }
+
+            // A new file or directory was created.
+            if (event.type === "create") {
+              const folder = findFolderById(sandboxFiles.files, sandboxDirectory) as TFolder;
+              const isDir = await isDirectory(containerFilePath);
+
+              const newItem = isDir
+                ? { id: sandboxFilePath, name: event.name, type: "folder", children: [] } as TFolder
+                : { id: sandboxFilePath, name: event.name, type: "file" } as TFile;
+
+              if (folder) {
+                // If the folder exists, add the new item (file/folder) as a child
+                folder.children.push(newItem);
+              } else {
+                // If folder doesn't exist, add the new item to the root
+                sandboxFiles.files.push(newItem);
+              }
+
+              if (!isDir) {
+                const fileData = await containers[data.sandboxId].files.read(containerFilePath);
+                const fileContents = typeof fileData === "string" ? fileData : "";
+                sandboxFiles.fileData.push({ id: sandboxFilePath, data: fileContents });
+              }
+
+              console.log(`Create ${sandboxFilePath}`);
+            }
+
+            // A file or directory was removed or renamed.
+            else if (event.type === "remove" || event.type == "rename") {
+              const folder = findFolderById(sandboxFiles.files, sandboxDirectory) as TFolder;
+              const isDir = await isDirectory(containerFilePath);
+
+              const isFileMatch = (file: TFolder | TFile | TFileData) => file.id === sandboxFilePath || file.id.startsWith(containerFilePath + '/');
+
+              if (folder) {
+                // Remove item from its parent folder
+                folder.children = folder.children.filter((file: TFolder | TFile) => !isFileMatch(file));
+              } else {
+                // Remove from the root if it's not inside a folder
+                sandboxFiles.files = sandboxFiles.files.filter((file: TFolder | TFile) => !isFileMatch(file));
+              }
+
+              // Also remove any corresponding file data
+              sandboxFiles.fileData = sandboxFiles.fileData.filter((file: TFileData) => !isFileMatch(file));
+
+              console.log(`Removed: ${sandboxFilePath}`);
+            }
+
+            // The contents of a file were changed.
+            else if (event.type === "write") {
+              const folder = findFolderById(sandboxFiles.files, sandboxDirectory) as TFolder;
+              const fileToWrite = sandboxFiles.fileData.find(file => file.id === sandboxFilePath);
+
+              if (fileToWrite) {
+                fileToWrite.data = await containers[data.sandboxId].files.read(containerFilePath);
+                console.log(`Write to ${sandboxFilePath}`);
+              } else {
+                // If the file is part of a folder structure, locate it and update its data
+                const fileInFolder = folder?.children.find(file => file.id === sandboxFilePath);
+                if (fileInFolder) {
+                  const fileData = await containers[data.sandboxId].files.read(containerFilePath);
+                  const fileContents = typeof fileData === "string" ? fileData : "";
+                  sandboxFiles.fileData.push({ id: sandboxFilePath, data: fileContents });
+                  console.log(`Write to ${sandboxFilePath}`);
+                }
+              }
+            }
+
+            // Tell the client to reload the file list
+            socket.emit("loaded", sandboxFiles.files);
+
+          } catch (error) {
+            console.error(`Error handling ${event.type} event for ${event.name}:`, error);
+          }
+        }, { "timeout": 0 } )
+      } catch (error) {
+        console.error(`Error watching filesystem:`, error);
+      }
+    };
+
+    // Watch the project directory
+    const handle = await watchDirectory(projectDirectory);
+    // Keep track of watch handlers to close later
+    if (handle) fileWatchers.push(handle);
+
+    // Watch all subdirectories of the project directory, but not deeper
+    // This also means directories created after the container is created won't be watched
+    const dirContent = await containerFiles.list(projectDirectory);
+    await Promise.all(dirContent.map(async (item : EntryInfo) => {
+      if (item.type === "dir") {
+        console.log("Watching " + item.path);
+        // Keep track of watch handlers to close later
+        const handle = await watchDirectory(item.path);
+        if (handle) fileWatchers.push(handle);
+      }
+    }))
+  
     socket.emit("loaded", sandboxFiles.files);
 
     socket.on("heartbeat", async () => {
@@ -263,7 +425,7 @@ io.on("connection", async (socket) => {
           path.posix.join(dirName, file.id),
           body
         );
-        fixPermissions();
+        fixPermissions(projectDirectory);
       } catch (e: any) {
         console.error("Error saving file:", e);
         io.emit("error", `Error: file saving. ${e.message ?? e}`);
@@ -285,7 +447,7 @@ io.on("connection", async (socket) => {
             path.posix.join(dirName, fileId),
             path.posix.join(dirName, newFileId)
           );
-          fixPermissions();
+          fixPermissions(projectDirectory);
 
           file.id = newFileId;
 
@@ -378,7 +540,7 @@ io.on("connection", async (socket) => {
           path.posix.join(dirName, id),
           ""
         );
-        fixPermissions();
+        fixPermissions(projectDirectory);
 
         sandboxFiles.files.push({
           id,
@@ -444,7 +606,7 @@ io.on("connection", async (socket) => {
           path.posix.join(dirName, fileId),
           path.posix.join(dirName, newFileId)
         );
-        fixPermissions();
+        fixPermissions(projectDirectory);
         await renameFile(fileId, newFileId, file.data);
       } catch (e: any) {
         console.error("Error renaming folder:", e);
@@ -669,6 +831,11 @@ io.on("connection", async (socket) => {
           connections[data.sandboxId]--;
         }
 
+        // Stop watching file changes in the container
+        Promise.all(fileWatchers.map(async (handle : WatchHandle) => {
+          await handle.close();
+        }));
+
         if (data.isOwner && connections[data.sandboxId] <= 0) {
           socket.broadcast.emit(
             "disableAccess",