User-agent: Scrapy
Allow: /

User-agent: *
Disallow: *?replytocom
Disallow: /downloads/
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /wp-content
Disallow: /wp-includes
Disallow: /wget/
Disallow: /httpd/
Disallow: /i/
Disallow: /f/
Disallow: /t/
Disallow: /c/
Disallow: /j/
Disallow: /pt/cgi-bin
Disallow: /pt/wp-admin
Disallow: /pt/wp-includes
Disallow: /pt/wp-content
Disallow: /pt/wget/
Disallow: /pt/httpd/
Disallow: /pt/i/
Disallow: /pt/f/
Disallow: /pt/t/
Disallow: /pt/c/
Disallow: /pt/j/
User-agent: *  
Allow: /wp-content/plugins/wp-minify/
User-agent: Mediapartners-Google
Allow: /
User-agent: Adsbot-Google
Allow: /
User-agent: Googlebot-Image
Allow: / 
User-agent: Googlebot-Mobile 
Allow: /
User-agent: Googlebot 
Allow: /