# Powered by MerchantPro
User-agent: *
Crawl-delay: 5
Sitemap: https://www.voltarom.ro/sitemap.xml
# No need to index this stuff
Disallow: /feed/
Disallow: /html/
Disallow: /scripts/
Disallow: /invoice
# Avoid duplicate content as it may do more harm than good
Disallow: /*?f=*
Disallow: /*?fb_xd_fragment
Disallow: /*?qty=*
# Disallow bad bots
User-agent: 008
User-agent: AhrefsBot
User-agent: Amazonbot
User-agent: Baiduspider
User-agent: BecomeBot
User-agent: BLEXBot
User-agent: CCBot
User-agent: EtaoSpider
User-agent: Exabot
User-agent: MJ12bot
User-agent: NPBot
User-agent: ShopWiki
User-agent: SMTBot
User-agent: sogouspider
User-agent: Sogou web spider
User-agent: Stanford
User-agent: Stanford Comp Sci
User-agent: Synthesio
User-agent: YandexBot
Disallow: /
Dissalow: /*?sort_by=
# Allow AI user-agents

# --- Perplexity ---

# Search Primary PerplexityBot crawler that indexes sites to build the Perplexity AI search engine.
User-agent: PerplexityBot
Allow: /

# Search Real-time fetcher used by Claude to retrieve cited URLs during AI chat sessions.
User-agent: ClaudeBot
Allow: /

# --- OPENAI ---
# Search (shows my webpages as links inside ChatGPT search). NOT used for model training.
User-agent: OAI-SearchBot
Allow: /

# User-driven browsing from ChatGPT and Custom GPTs. Acts after a human click.
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
Allow: /

# --- GOOGLE (Gemini) ---
User-agent: Google-Extended
Allow: /