User-agent: * # Disallows aimed at Google. Keep it from running search queries, except for a few. Allow: /search/results/?q=github$ Allow: /search/results/?q=bitcoin$ Allow: /search/results/?q=css$ Disallow: /search/results/ # Disallows aimed at Mastodon/GNUSocial. My experimental node shuttered in 2018, please stop hitting me, thanks. Disallow: /api/ Disallow: /nodeinfo/ Disallow: /main/ # Disallows aimed at other bots Disallow: /*wlwmanifest.xml # Allow only specific .well-known locations, block the rest Allow: /.well-known/openpgpkey/ Disallow: /.well-known/