# /public_html/robots.txt (Used by Cantek)
# General rules for all bots
User-agent: *
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /?s=
Allow: /wp-admin/admin-ajax.php

# Explicit allow for major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

# Block known aggressive or non-beneficial bots
User-agent: AhrefsBot
Disallow: /
# Crawl-delay: 10

User-agent: SEMrushBot
Disallow: /
Crawl-delay: 10

User-agent: MJ12Bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Sitemap location
Sitemap: https://cantekamerica.com/sitemap_index.xml