# Allow social media crawlers for link previews User-agent: Twitterbot Allow: / User-agent: facebookexternalhit Allow: / User-agent: LinkedInBot Allow: / User-agent: WhatsApp Allow: / User-agent: Slackbot Allow: / # Allow search engines User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / # Allow AI search agents to fetch public, answer-ready pages User-agent: GPTBot Allow: / Allow: /llms.txt Allow: /llms-full.txt User-agent: ChatGPT-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /dashboard Disallow: /api Disallow: /auth Disallow: /_next User-agent: CCBot Allow: / Allow: /llms.txt Allow: /llms-full.txt User-agent: ClaudeBot Allow: / Allow: /llms.txt Allow: /llms-full.txt User-agent: Claude-Web Allow: / Allow: /llms.txt Allow: /llms-full.txt User-agent: Claude-SearchBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /dashboard Disallow: /api Disallow: /auth Disallow: /_next User-agent: Claude-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /dashboard Disallow: /api Disallow: /auth Disallow: /_next User-agent: OAI-SearchBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /dashboard Disallow: /api Disallow: /auth Disallow: /_next User-agent: anthropic-ai Allow: / Allow: /llms.txt Allow: /llms-full.txt User-agent: PerplexityBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /dashboard Disallow: /api Disallow: /auth Disallow: /_next User-agent: Perplexity-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /dashboard Disallow: /api Disallow: /auth Disallow: /_next User-agent: Google-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt # Default rule for all bots User-agent: * Allow: / Allow: /llms.txt Allow: /llms-full.txt # Disallow admin/internal paths if any Disallow: /api/ Disallow: /_next/ Disallow: /dashboard Disallow: /auth Disallow: /admin/ # Sitemap Sitemap: https://autoshort.org/sitemap.xml