seo: sitemap, robots, og tags

frontend/src/app/robots.ts

@@ -0,0 +1,14 @@
+import { MetadataRoute } from "next";
+
+const BASE_URL = process.env.NEXT_PUBLIC_SITE_URL || "https://codetutor.example.com";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: "*",
+      allow: "/",
+      disallow: ["/api/"],
+    },
+    sitemap: `${BASE_URL}/sitemap.xml`,
+  };
+}