# Robots.txt for Doherty Home Loans
# Florida Mortgage Lender - Louis Doherty NMLS #60093

# Allow all crawlers to access public content
User-agent: *
Allow: /

# Disallow test/staging content (if any)
Disallow: /test/
Disallow: /staging/
Disallow: /dev/
Disallow: /_redirects
Disallow: /*.md$

# Disallow query parameter variations (canonical versions preferred)
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*

# Disallow duplicate content
Disallow: /optimized-services-section.html

# Disallow widget-only pages (embedded elsewhere)
Disallow: /mortgage-calculator-widget.html

# Allow important pages explicitly
Allow: /index.html
Allow: /about.html
Allow: /new-home-purchases.html
Allow: /refinancing.html
Allow: /fha-loans-florida.html
Allow: /jumbo-loans.html
Allow: /investment-properties.html
Allow: /self-employed-loans-florida.html
Allow: /mortgage-calculator.html
Allow: /privacy-policy.html
Allow: /the-home-finance-guide.html
Allow: /sitemap*.xml
Allow: /html-sitemap.html

# Crawl-delay (optional, remove if causing issues)
# Crawl-delay: 1

# Sitemap locations
Sitemap: https://dohertyhomeloans.com/sitemap.xml
Sitemap: https://dohertyhomeloans.com/sitemap-main.xml
Sitemap: https://dohertyhomeloans.com/sitemap-services.xml
Sitemap: https://dohertyhomeloans.com/sitemap-blog.xml
Sitemap: https://dohertyhomeloans.com/sitemap-tools.xml
Sitemap: https://dohertyhomeloans.com/sitemap-info.xml
Sitemap: https://dohertyhomeloans.com/sitemap-cities.xml

# Host preference (choose one)
# Host: https://dohertyhomeloans.com

# Additional crawler-specific rules
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /images/

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Block bad bots (optional)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

# Last updated: 2025-10-23