Create robots.txt

ethanniser · Oct 22, 2024 · e1d6247 · e1d6247
1 parent e1f993c
commit e1d6247
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/src/app/robots.txt b/src/app/robots.txt
@@ -0,0 +1,15 @@
+# Block all crawlers from accessing the site
+User-agent: *
+Disallow: /
+
+# Allow Twitter to fetch Open Graph data
+User-agent: Twitterbot
+Allow: /
+
+# Allow Facebook to fetch Open Graph data
+User-agent: facebookexternalhit
+Allow: /
+
+# Allow LinkedIn to fetch Open Graph data
+User-agent: LinkedInBot
+Allow: /