diff --git a/src/app/robots.txt b/src/app/robots.txt new file mode 100644 index 0000000..383664c --- /dev/null +++ b/src/app/robots.txt @@ -0,0 +1,15 @@ +# Block all crawlers from accessing the site +User-agent: * +Disallow: / + +# Allow Twitter to fetch Open Graph data +User-agent: Twitterbot +Allow: / + +# Allow Facebook to fetch Open Graph data +User-agent: facebookexternalhit +Allow: / + +# Allow LinkedIn to fetch Open Graph data +User-agent: LinkedInBot +Allow: /