# WorldYellowPages — robots policy
# Public listing pages are crawlable; admin, search, and Symfony internals are not.
# SEO-tool crawlers that only consume budget are blocked — EXCEPT SemrushBot
# (the operator uses Semrush; it gets a gentler crawl-delay instead).

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /_profiler/
Disallow: /_wdt/
Disallow: /_fragment
Disallow: /search
Disallow: /search?
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Disallow: /admin/
Disallow: /search
Crawl-delay: 5

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

Sitemap: /sitemap.xml