From 39b779271733551d439dc7680f64b3955f658d05 Mon Sep 17 00:00:00 2001
From: Taken <taken@mairimashita.org>
Date: Wed, 6 Aug 2025 22:50:53 +0200
Subject: [PATCH] Updated robots

---
 src/app/robots.ts | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/src/app/robots.ts b/src/app/robots.ts
index b0a6bae..b6f28d0 100644
--- a/src/app/robots.ts
+++ b/src/app/robots.ts
@@ -1,12 +1,28 @@
+import { getAllUrls } from "@/lib/db/urls"
 import type { MetadataRoute } from "next"
 
-export default function robots(): MetadataRoute.Robots {
+export default async function robots(): Promise<MetadataRoute.Robots> {
+    const slugs = (await getAllUrls()).map(s => `/r/${s.slug}`)
+
+    const allowed = ["/"]
+    const allowedBots = ["Discordbot", "Googlebot"]
+
+    const mainRules: MetadataRoute.Robots["rules"] = [{
+        userAgent: allowedBots,
+        allow: allowed,
+        crawlDelay: 60
+    }]
+
+    const dynamicRules: MetadataRoute.Robots["rules"] = [{
+        userAgent: allowedBots,
+        allow: slugs,
+        crawlDelay: 1
+    }]
+
     return {
-        rules: {
+        rules: [...mainRules, ...dynamicRules, {
             userAgent: "*",
-            allow: "/",
-            disallow: ["/api", "/dasboard", "/sign-in"],
-            crawlDelay: 1
-        }
+            disallow: "*"
+        }]
     }
 }