# For all web crawlers
User-agent: *

# Disallow API endpoints. They are for the site's functionality 
# and do not contain content that should appear in search results.
Disallow: /api/
Disallow: /autocomplete
Disallow: /results

# Disallow internal data files from being crawled.
# These files are for the application's use and not for public consumption.
Disallow: /config.json
Disallow: /scrape_status.json
Disallow: /series.json
Disallow: /site_languages.json

# Disallow crawling of potential admin areas.
Disallow: /admin/

# For Google's AdSense crawlers, allow full access to ensure ad relevance.
# An empty "Disallow" means nothing is disallowed for this user-agent.
User-agent: Mediapartners-Google
Disallow:

User-agent: AdsBot-Google
Disallow:

# Location of the sitemap. This helps crawlers find all your important pages.
Sitemap: https://freeseriesfinder.com/sitemap.xml