#This file will allow filtering of directories to not allow robots
#to spider certain directories, see examples below

User-agent: *
disallow: /cgi-bin
disallow: /wp-admin
disallow: /wp-includes
disallow: /wp-login.php