User-agent: *
Disallow: /api
Disallow: /cgi-bin
Disallow: /common

User-agent: ia_archiver
Disallow: /

User-agent: psbot
Disallow: /