From 2f88ff6d58752abd31d029bc54212788882b7a6b Mon Sep 17 00:00:00 2001
From: James Murdza <james@jamesmurdza.com>
Date: Sun, 15 Sep 2024 10:29:23 -0700
Subject: [PATCH] feat: speed up new project creation by copying files
 concurrently

---
 backend/storage/package-lock.json | 32 ++++++++++++++++++++++---------
 backend/storage/package.json      |  1 +
 backend/storage/src/index.ts      | 19 ++++++++++--------
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/backend/storage/package-lock.json b/backend/storage/package-lock.json
index 3b9a47e..8304e66 100644
--- a/backend/storage/package-lock.json
+++ b/backend/storage/package-lock.json
@@ -8,6 +8,7 @@
       "name": "storage",
       "version": "0.0.0",
       "dependencies": {
+        "p-limit": "^6.1.0",
         "zod": "^3.23.4"
       },
       "devDependencies": {
@@ -894,6 +895,21 @@
         "url": "https://opencollective.com/vitest"
       }
     },
+    "node_modules/@vitest/runner/node_modules/p-limit": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz",
+      "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==",
+      "dev": true,
+      "dependencies": {
+        "yocto-queue": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/@vitest/snapshot": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.3.0.tgz",
@@ -1766,12 +1782,11 @@
       }
     },
     "node_modules/p-limit": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz",
-      "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==",
-      "dev": true,
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz",
+      "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==",
       "dependencies": {
-        "yocto-queue": "^1.0.0"
+        "yocto-queue": "^1.1.1"
       },
       "engines": {
         "node": ">=18"
@@ -2970,10 +2985,9 @@
       "dev": true
     },
     "node_modules/yocto-queue": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz",
-      "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==",
-      "dev": true,
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz",
+      "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==",
       "engines": {
         "node": ">=12.20"
       },
diff --git a/backend/storage/package.json b/backend/storage/package.json
index 8b210b7..3f5c39f 100644
--- a/backend/storage/package.json
+++ b/backend/storage/package.json
@@ -17,6 +17,7 @@
     "wrangler": "^3.0.0"
   },
   "dependencies": {
+    "p-limit": "^6.1.0",
     "zod": "^3.23.4"
   }
 }
diff --git a/backend/storage/src/index.ts b/backend/storage/src/index.ts
index c57e7e7..c9b371e 100644
--- a/backend/storage/src/index.ts
+++ b/backend/storage/src/index.ts
@@ -1,7 +1,9 @@
 import { z } from "zod"
+import pLimit from 'p-limit';
 
 export interface Env {
 	R2: R2Bucket
+	Templates: R2Bucket
 	KEY: string
 }
 
@@ -144,17 +146,18 @@ export default {
 
 			console.log(`Copying template: ${type}`);
 
-			const templateDirectory = `templates/${type}`;
-
 			// List all objects under the directory
-			const { objects } = await env.R2.list({ prefix: templateDirectory });
+			const { objects } = await env.Templates.list({ prefix: type });
 
-			// Copy each object to the new directory
-			for (const { key } of objects) {
-				const destinationKey = key.replace(templateDirectory, `projects/${sandboxId}`);
-				const fileBody = await env.R2.get(key).then(res => res?.body ?? "");
+			// Copy each object to the new directory with a 5 concurrency limit
+			const limit = pLimit(5);
+			await Promise.all(objects.map(({ key }) =>
+			  limit(async () => {
+				const destinationKey = key.replace(type, `projects/${sandboxId}`);
+				const fileBody = await env.Templates.get(key).then(res => res?.body ?? "");
 				await env.R2.put(destinationKey, fileBody);
-			}
+			  })
+			));
 
 			return success
 		} else {