# Toolsy.lt - Įrankių nuomos platforma
# Lithuania's First Peer-to-Peer Tool Rental Marketplace
# https://toolsy.lt
# Last updated: 2025-12-20

# Default rules for all crawlers
User-agent: *
Allow: /
Crawl-delay: 1
Disallow: /*?*search=*

# Block admin and private areas
Disallow: /admin
Disallow: /paskyra
Disallow: /mano-garazas
Disallow: /mano-rezervacijos
Disallow: /pirkiniai
Disallow: /sukurti-skelbima
Disallow: /redaguoti-skelbima/

# Google Search Bot - no delay needed
User-agent: Googlebot
Allow: /
Disallow: /*?*search=*
Disallow: /admin
Disallow: /paskyra

# Bing Bot
User-agent: Bingbot
Allow: /
Crawl-delay: 2
Disallow: /admin
Disallow: /paskyra

# Yandex Bot
User-agent: Yandex
Allow: /
Crawl-delay: 3
Disallow: /admin
Disallow: /paskyra

# AI Chatbots - Welcome with moderate rate
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Crawl-delay: 2

User-agent: Anthropic-AI
Allow: /
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

User-agent: Google-Extended
Allow: /
Crawl-delay: 2

User-agent: Applebot-Extended
Allow: /
Crawl-delay: 2

User-agent: CCBot
Allow: /
Crawl-delay: 3

User-agent: Cohere-ai
Allow: /
Crawl-delay: 2

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2

User-agent: Bytespider
Allow: /
Crawl-delay: 5

User-agent: YouBot
Allow: /
Crawl-delay: 2

# Social Media Bots - fast access for previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

# Block aggressive/unwanted bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemap location
Sitemap: https://toolsy.lt/sitemap.xml

# AI Information Files
# For AI models seeking information about Toolsy.lt:
# - https://toolsy.lt/llms.txt (summary)
# - https://toolsy.lt/llms-full.txt (comprehensive)
# - https://toolsy.lt/ai.txt (crawler policy)
#
# API for real-time data:
# GET https://zkbmqqdbvchfjwsocypn.supabase.co/functions/v1/ai-tools-api