Sitemap: https://kolappan.com/sitemap.xml

User-Agent: *
Allow: /
Disallow: /tags/
Disallow: / search/
Crawl-delay: 30

# Amazon bots
User-Agent: Amazonbot
User-Agent: AmazonAdBot
User-Agent: FriendlyCrawler
Disallow: /

# Google Bot
# Ref: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
User-Agent: Google-Extended
User-Agent: Googlebot-Image
User-Agent: Googlebot-News
User-Agent: Googlebot-Video
User-Agent: Storebot-Google
User-Agent: GoogleOther
User-Agent: GoogleOther-Image
User-Agent: GoogleOther-Video
User-Agent: AdsBot-Google
User-Agent: AdsBot-Google-Mobile
User-Agent: Mediapartners-Google
Disallow: /

# AI & LLMs
User-Agent: Applebot-Extended
User-Agent: GPTBot
User-Agent: anthropic-ai
User-Agent: Bytespider
User-agent: CCBot
User-Agent: Claude-Web
User-Agent: ClaudeBot
User-Agent: cohere-ai
User-Agent: Diffbot
User-Agent: FacebookBot
User-Agent: ImagesiftBot
User-Agent: OmigiliBot
User-Agent: Omigili
User-Agent: PerplexityBot
User-Agent: scoop.it
Disallow: /

# Ad bots
User-Agent: adidxbot
User-Agent: CriteoBot
User-Agent: grapeshot
User-Agent: proximic
User-Agent: Taboolabot
Disallow: /

# SEO & Analytics
User-Agent: AddThis.com
User-Agent: AhrefsBot
User-Agent: AwarioBot
User-Agent: AwarioSmartBot
User-Agent: AwarioRssBot
User-Agent: Barkrowler
User-Agent: botify
User-Agent: BLEXBot
User-Agent: DataForSEO
User-Agent: dotbot
User-Agent: Embedly
User-Agent: GingerCrawler
User-Agent: PiplBot
User-Agent: SemrushBot
User-Agent: WebReaper
Disallow: /

# chinese search engines
# baidu.com
User-Agent: Baiduspider
User-Agent: Baiduspider-image
User-Agent: Baiduspider-news
User-Agent: Baiduspider-video
Disallow: /
# so.com
User-Agent: 360Spider
User-Agent: 360Spider-Image
User-Agent: 360Spider-Video
User-Agent: HaoSouSpider
Disallow: /
# jike.com / chinaso.com
User-Agent: JikeSpider
Disallow: /
# sogou.com
User-Agent: Sogou blog
User-Agent: Sogou inst spider
User-Agent: Sogou News Spider
User-Agent: Sogou Orion spider
User-Agent: Sogou spider2
User-Agent: Sogou web spider
Disallow: /
# soso.com
User-Agent: Sosospider
Disallow: /