Updated files

src/app/robots.ts

@@ -1,31 +1,12 @@
-import { getAllUrls } from "@/lib/db/urls"
 import type { MetadataRoute } from "next"
 
-export const dynamic = "force-dynamic"
-
-export default async function robots(): Promise<MetadataRoute.Robots> {
-    const urls = await getAllUrls()
-
-    const crawlableUrls = urls.map(u => {
-        if (u.crawlable) {
-            return `/r/${u.slug}`
-        }
-    }).filter(v => typeof v === "string")
-
+export default function robots(): MetadataRoute.Robots {
     return {
-        rules: [
-            {
-                userAgent: "*",
-                allow: "/",
-                disallow: ["/api", "/dasboard", "/sign-in"],
-                crawlDelay: 1
-            },
-            {
-                userAgent: "*",
-                disallow: "/r/",
-                allow: crawlableUrls,
-                crawlDelay: 1
-            }
-        ]
+        rules: {
+            userAgent: "*",
+            allow: "/",
+            disallow: ["/api", "/dasboard", "/sign-in"],
+            crawlDelay: 1
+        }
     }
 }