Update robots.txt

ethanniser · Oct 22, 2024 · 393bdcc · 393bdcc
1 parent 6f084ef
commit 393bdcc
Showing 1 changed file with 10 additions and 7 deletions.
diff --git a/src/app/robots.txt b/src/app/robots.txt
@@ -2,18 +2,21 @@
 User-agent: *
 Disallow: /products
 
-# Allow crawling of the homepage
-User-agent: *
-Allow: /
-
-# Allow Twitter to fetch Open Graph data
+# Allow Twitter to fetch Open Graph data from /products
 User-agent: Twitterbot
+Allow: /products
 Allow: /
 
-# Allow Facebook to fetch Open Graph data
+# Allow Facebook to fetch Open Graph data from /products
 User-agent: facebookexternalhit
+Allow: /products
 Allow: /
 
-# Allow LinkedIn to fetch Open Graph data
+# Allow LinkedIn to fetch Open Graph data from /products
 User-agent: LinkedInBot
+Allow: /products
+Allow: /
+
+# Allow crawling of the homepage for all crawlers
+User-agent: *
 Allow: /