Updated robots

src/app/robots.ts

@@ -1,12 +1,28 @@
+import { getAllUrls } from "@/lib/db/urls"
 import type { MetadataRoute } from "next"
 
-export default function robots(): MetadataRoute.Robots {
+export default async function robots(): Promise<MetadataRoute.Robots> {
-    return {
+    const slugs = (await getAllUrls()).map(s => `/r/${s.slug}`)
-        rules: {
+
-            userAgent: "*",
+    const allowed = ["/"]
-            allow: "/",
+    const allowedBots = ["Discordbot", "Googlebot"]
-            disallow: ["/api", "/dasboard", "/sign-in"],
+
+    const mainRules: MetadataRoute.Robots["rules"] = [{
+        userAgent: allowedBots,
+        allow: allowed,
+        crawlDelay: 60
+    }]
+
+    const dynamicRules: MetadataRoute.Robots["rules"] = [{
+        userAgent: allowedBots,
+        allow: slugs,
         crawlDelay: 1
-        }
+    }]
+
+    return {
+        rules: [...mainRules, ...dynamicRules, {
+            userAgent: "*",
+            disallow: "*"
+        }]
     }
 }