User-agent: *
Allow: /$

# Allow crawling of important content
Allow: /sitemap-1.xml
Allow: /sitemap_index.xml
Allow: /post-sitemap.xml
Allow: /page-sitemap.xml
Allow: /wp-content/uploads/

# Disallow crawling of certain directories
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php

# Block access to sensitive files
Disallow: /license.txt
Disallow: /readme.html

# Block search page and query parameters
Disallow: /?s=
Disallow: /*?

# Block specific paths and pages
Disallow: /privacy-policy/
Disallow: /terms-and-conditions/

# Block staging or development environment
Disallow: /dev/
Disallow: /staging/

# Crawling rate limit
Crawl-delay: 5

# Set crawl budget
# Crawl-delay: 10

# Set a custom crawl rate (1 request every 10 seconds)
# Crawl-delay: 10

# Set a specific user-agent crawl rate (Googlebot example)
# User-agent: Googlebot
# Crawl-delay: 10

Sitemap: https://dealershipforlife.com/sitemap.xml
Sitemap: https://dealershipforlife.com/news-sitemap.xml

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-json/
Disallow: /?rest_route=

User-agent: AdsBot
Disallow: /

Sitemap: https://dealershipforlife.com/sitemap_index.xml
Sitemap: https://dealershipforlife.com/sitemap_1.xml
# ---------------------------
# END YOAST BLOCK