From 474e08516dd8259471e5dfa27661d1f9c0815529 Mon Sep 17 00:00:00 2001 From: snxraven Date: Thu, 26 Sep 2024 04:48:45 -0400 Subject: [PATCH] revert f64c5df9f3baafabfddddb8a945a665c7da0c226 revert revert 388d4891817790845857218b051e8eb901ec5573 revert Remove dynamic robots.txt and move to static txt --- app.js | 27 --------------------------- me/robots.md | 5 ----- public/robots.txt | 3 +++ 3 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 me/robots.md create mode 100644 public/robots.txt diff --git a/app.js b/app.js index e79758d..508f6c2 100644 --- a/app.js +++ b/app.js @@ -313,33 +313,6 @@ app.get('/rss', (req, res) => { res.send(rssFeed); }); - -// Function to parse markdown and format as robots.txt -function generateRobotsTxt(env) { - const robotsConfigFile = path.join(__dirname, 'me', 'robots.md'); - const markdownContent = fs.readFileSync(robotsConfigFile, 'utf-8'); - const sections = markdownContent.split('###').map(section => section.trim()).filter(Boolean); - - let configSection = sections.find(section => section.startsWith(env.charAt(0).toUpperCase() + env.slice(1))); - - if (configSection) { - configSection = configSection.split('\n').slice(1); // Remove the section title - return configSection.map(line => line.replace('- ', '')).join('\n'); // Remove Markdown list dashes - } - - // Default fallback if no matching environment is found - return 'User-agent: *\nDisallow: /'; -} - -// Robots.txt Route -app.get('/robots.txt', (req, res) => { - const env = process.env.NODE_ENV || 'development'; // Default to 'development' if not set - const robotsContent = generateRobotsTxt(env); - - res.type('text/plain'); - res.send(robotsContent); -}); - // Create a URL object from the environment variable const blog_URL = new URL(process.env.BLOG_URL); diff --git a/me/robots.md b/me/robots.md deleted file mode 100644 index 0097fb6..0000000 --- a/me/robots.md +++ /dev/null @@ -1,5 +0,0 @@ -### Production - -- User-agent: * -- Allow: / -- Sitemap: https://raven-scott.fyi/sitemap.xml \ No newline at end of file diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..a8407d4 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Allow: / +Sitemap: https://raven-scott.fyi/sitemap.xml \ No newline at end of file