# START YOAST BLOCK
# ---------------------------
# User-agent: *
# Disallow:

Sitemap: https://samuelautomation.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

# Default search crawler policy: allow public site discovery.
User-agent: *
Allow: /

# WordPress/API utility endpoint often needed by themes and forms.
Allow: /wp-admin/admin-ajax.php

# Keep low-value, duplicate, private, or infinite URL spaces out of crawl paths.
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /search
Disallow: /*?s=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?sessionid=

# AI-search discovery: allow public content, but keep private/duplicate paths blocked.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /search
Disallow: /*?s=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?sessionid=

# AI-search discovery: allow public content, but keep private/duplicate paths blocked.
User-agent: Claude-User
User-agent: Claude-SearchBot
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /search
Disallow: /*?s=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?sessionid=

# OpenAI training crawler: blocked by default. Change to Allow: / if Samuel's policy permits model-training use.
User-agent: GPTBot
Disallow: /

# Google AI usage control. This does not block Google Search crawling.
# Change to Allow: / if Samuel's policy permits Gemini/Vertex AI usage.
User-agent: Google-Extended
Disallow: /

# Anthropic crawler. Change to Allow: / if Samuel's policy permits it.
User-agent: ClaudeBot
Disallow: /

# Common Crawl is often used by downstream AI/data systems.
# Change to Allow: / if open dataset inclusion is desired.
User-agent: CCBot
Disallow: /