# 01/06/2019 Robots.txt for TomaChollos.com
#
# Allow
# Evita bloqueos de CSS y JS.
User-Agent: *
Allow: /*.js$
Allow: /*.css$
Allow: */modules/*.css
Allow: /*.css
Allow: /*.js
Allow: /wp-admin/*.js
Allow: /wp-admin/*.css
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js/jquery/jquery.js
Allow: /wp-includes/js/jquery/jquery.min.js
Allow: /wp-includes/js/*
Allow: /wp-content/themes/kadence/assets/css/*.css
Allow: /wp-content/themes/kadence/assets/css/*

#	Seguimos por contenido adjunto
User-Agent: *
Allow: /wp-content/uploads/
Allow: /wp-content/litespeed/cssjs/*.css
Allow: /wp-content/litespeed/cssjs/*.js
Allow: /wp-content/webp-express/webp-images/doc-root/wp-content/
Allow: /wp-content/webp-express/webp-images/doc-root/wp-content/*
Disallow: /wp-content/plugins/
Disallow: /wp-includes/

#	Seguimos por contenido admin y redirects
User-Agent: *
Disallow: /wp-admin/
Disallow: /redirect-to

#	Otra info y búsquedas
User-Agent: *
Disallow: /wp-
Disallow: /?s=
Disallow: */?s=
Disallow: /*?s=
Disallow: /search
Disallow: /wp-content/
Disallow: /archives/
Disallow: /cgi-bin
Disallow: /suscripciones-a-comentarios/*
Disallow: /xmlrpc.php
Disallow: /?attachment_id*
Disallow: /go/


#   Paginación
#User-Agent: *
#Disallow: /*/page/
#Disallow: /tag/*/page/
#Disallow: /page/

# No indexamos copias de seguridad
User-Agent: *
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*.gz$
Disallow: /*.tar$
Disallow: /*.svn$

# Autorizamos el feed general para Google Blogsearch
# Pero impedimos que permalink/feed/ sea indexado ya que el
# feed con los comentarios suele posicionarse en lugar de
# la entrada y desorienta a los usuarios.
#
# Lo mismo con URLs terminadas en /trackback/ que sólo
# sirven como Trackback URI (y son contenido duplicado).
#
User-Agent: *
Allow: /feed/$
Disallow: /feed/
Disallow: /comments/feed
Disallow: /*/feed/
Disallow: /*/feed/$
Disallow: /*/feed/rss/$
Disallow: /*/trackback/$
Disallow: /*/*/feed/$
Disallow: /*/*/feed/rss/$
Disallow: /*/*/trackback/$
Disallow: /*/*/*/feed/$
Disallow: /*/*/*/feed/rss/$
Disallow: /*/*/*/trackback/$
Disallow: /trackback/
Disallow: /tag/*/feed/
Disallow: */out
Disallow: /*/*/*/feed.xml

#	Ahora el sitemap
User-Agent: *
Sitemap: https://www.tomachollos.com/sitemap_index.xml

#   Lista de bots permitidos
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
User-agent: Adsbot-Google
Allow: /
User-agent: Googlebot-Mobile
Allow: /

# Lista de bots que suelen respetar el robots.txt pero rara
# vez hacen un buen uso del sitio y abusan bastante…
#
User-agent: MSIECrawler
Disallow: /
Crawl-delay: 10
User-agent: WebCopier
Disallow: /
Crawl-delay: 10
User-agent: HTTrack
Disallow: /
Crawl-delay: 10
User-agent: Microsoft.URL.Control
Disallow: /
Crawl-delay: 10
User-agent: libwww
Disallow: /
Crawl-delay: 10

# Slurp (Yahoo!), Noxtrum y el bot de MSN a veces tienen
# idas de pinza, toca decirles que reduzcan la marcha.
#
User-agent: noxtrumbot
Crawl-delay: 20
User-agent: msnbot
Crawl-delay: 20
User-agent: Slurp
Crawl-delay: 20

#	CuteStat no es muy respetuoso con los privacy y otros son poco útiles.
#
User-agent: CuteStat
Disallow: /
User-agent: Orthogaffe 
Disallow: / 
User-agent: UbiCrawler 
Disallow: / 
User-agent: DOC 
Disallow: / 
User-agent: Zao 
Disallow: / 
User-agent: sitecheck.internetseer.com 
Disallow: / 
User-agent: Zealbot 
Disallow: / 
User-agent: MSIECrawler 
Disallow: / 
User-agent: SiteSnagger 
Disallow: / 
User-agent: WebStripper 
Disallow: / 
User-agent: WebCopier 
Disallow: / 
User-agent: Fetch 
Disallow: / 
User-agent: Offline Explorer 
Disallow: / 
User-agent: Teleport 
Disallow: / 
User-agent: TeleportPro 
Disallow: / 
User-agent: WebZIP 
Disallow: / 
User-agent: linko 
Disallow: / 
User-agent: HTTrack 
Disallow: / 
User-agent: Microsoft.URL.Control 
Disallow: / 
User-agent: Xenu 
Disallow: / 
User-agent: larbin 
Disallow: / 
User-agent: libwww 
Disallow: / 
User-agent: ZyBORG 
Disallow: / 
User-agent: Download Ninja 
Disallow: / 
User-agent: wget 
Disallow: / 
User-agent: grub-client 
Disallow: / 
User-agent: k2spider 
Disallow: / 
User-agent: NPBot 
Disallow: / 
User-agent: WebReaper 
Disallow: /