From e1d624798e16b87dc6f19bc71f0d7e4af944047e Mon Sep 17 00:00:00 2001
From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com>
Date: Tue, 22 Oct 2024 10:52:36 -0700
Subject: [PATCH] Create robots.txt

---
 src/app/robots.txt | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 src/app/robots.txt

diff --git a/src/app/robots.txt b/src/app/robots.txt
new file mode 100644
index 0000000..383664c
--- /dev/null
+++ b/src/app/robots.txt
@@ -0,0 +1,15 @@
+# Block all crawlers from accessing the site
+User-agent: *
+Disallow: /
+
+# Allow Twitter to fetch Open Graph data
+User-agent: Twitterbot
+Allow: /
+
+# Allow Facebook to fetch Open Graph data
+User-agent: facebookexternalhit
+Allow: /
+
+# Allow LinkedIn to fetch Open Graph data
+User-agent: LinkedInBot
+Allow: /