Updated robots

src/app/robots.ts

@@ -1,12 +1,28 @@
+import { getAllUrls } from "@/lib/db/urls"
 import type { MetadataRoute } from "next"
 
-export default function robots(): MetadataRoute.Robots {
+export default async function robots(): Promise<MetadataRoute.Robots> {
+    const slugs = (await getAllUrls()).map(s => `/r/${s.slug}`)
+
+    const allowed = ["/"]
+    const allowedBots = ["Discordbot", "Googlebot"]
+
+    const mainRules: MetadataRoute.Robots["rules"] = [{
+        userAgent: allowedBots,
+        allow: allowed,
+        crawlDelay: 60
+    }]
+
+    const dynamicRules: MetadataRoute.Robots["rules"] = [{
+        userAgent: allowedBots,
+        allow: slugs,
+        crawlDelay: 1
+    }]
+
     return {
-        rules: {
+        rules: [...mainRules, ...dynamicRules, {
             userAgent: "*",
-            allow: "/",
-            disallow: ["/api", "/dasboard", "/sign-in"],
-            crawlDelay: 1
-        }
+            disallow: "*"
+        }]
     }
 }