# Award Watch. Every page is open to search engines, in English and in French
# (?lang=fr). The JSON API and the response form are not pages to index.
#
# Crawling the profile and notice pages gains nothing the nightly bulk files
# do not hold: see /llms.txt for the files and their licence. The bulk files
# under /downloads/ may be fetched (ten a minute per address).
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
Disallow: /organizations/*/respond
# Search results: every query would be a page, and each one reads the search
# database. The directory and the sitemaps reach every profile.
Disallow: /search
# A filtered tender list, a paged or filtered ranking, a day of new notices
# and a correction form opened from a profile (?organization=) each name the
# plain page as their canonical URL, so crawlers need not walk the query
# forms. The plain pages and their French versions (?lang=fr) stay open.
Disallow: /tenders?
Allow: /tenders?lang=fr$
Disallow: /tenders/new?
Allow: /tenders/new?lang=fr$
Disallow: /rankings?
Allow: /rankings?lang=fr$
Disallow: /corrections?
Allow: /corrections?lang=fr$
Crawl-delay: 5

# Bing honours Crawl-delay, and five seconds would cap it far below the
# number of profile and notice pages. A crawler reads only the group that
# names it, so Bingbot's group repeats the rules without the delay.
User-agent: bingbot
Allow: /
Disallow: /api/
Disallow: /organizations/*/respond
Disallow: /search
Disallow: /tenders?
Allow: /tenders?lang=fr$
Disallow: /tenders/new?
Allow: /tenders/new?lang=fr$
Disallow: /rankings?
Allow: /rankings?lang=fr$
Disallow: /corrections?
Allow: /corrections?lang=fr$

# Crawlers that gather pages for AI training or large-scale scraping are not
# needed to find the records (the data is downloadable on the Data page) and
# at their pace they use up the request allowance that readers depend on.
# Assistants fetching a single page for a person are not listed here.
User-agent: ShapBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: GPTBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: omgili
User-agent: PetalBot
User-agent: DataForSeoBot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
Disallow: /

Sitemap: https://awardwatch.ca/sitemap.xml