# robots.txt for Rinda.ai
# https://rinda.ai/robots.txt
#
# IMPORTANT — read before editing:
# A crawler obeys ONLY the single most-specific User-agent group that matches
# its token; it does NOT inherit rules from the `User-agent: *` group. So every
# named group below MUST repeat the same Disallow rules as `*`, otherwise that
# bot is silently allowed to crawl /admin/, /api/ and /blog/cms/. Keep the
# DISALLOW block in sync across ALL groups.

User-agent: *
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# SEO Pages - High Priority
Allow: /alternatives/
Allow: /best/
Allow: /solutions/

# Content Pages
Allow: /blog/
Allow: /glossary/
Allow: /industry/
Allow: /country/
Allow: /export-guide/

# Core Pages
Allow: /pricing
Allow: /partners
Allow: /partners-plan
# /all-in-one 은 여기 없다 — 페이지가 존재한 적이 없고 실측 404 다(2026-08-25).
# 없는 경로를 Allow 로 광고하면 크롤러가 404 를 반복해서 긁는다.

# AI/GEO assets (explicit allow for clarity, default Allow:/ already covers)
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms-ja.txt
Allow: /llms-th.txt
Allow: /llms-zh.txt

# Block admin, API and CMS routes (kept in sync across every group below)
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Google specific
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/
Crawl-delay: 0

# Bing specific
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/
Crawl-delay: 1

# Naver (Yeti) — 한국 1위 엔진. 명명 그룹이 없으면 `*` 그룹의 Crawl-delay: 1
# 을 받는다(크롤러는 가장 구체적인 그룹 하나만 따른다 — 파일 상단 주석 참조).
# 기본 로케일(ko)의 주력 엔진을 스로틀할 이유가 없다 — Googlebot 과 동급 대우.
User-agent: Yeti
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/
Crawl-delay: 0

# Daum (Daumoa) — 같은 이유.
User-agent: Daumoa
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/
Crawl-delay: 0

# AI Search Engine Crawlers - allowed for GEO, but still blocked from
# admin/api/cms (same non-public surfaces hidden from every crawler).
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: meta-externalagent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: DuckAssistBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: MistralAI-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

# Chinese search + AI crawlers.
# Mainland China is a real organic-traffic source for /zh-CN, and these bots do
# NOT inherit the `User-agent: *` group — without an explicit group they fall
# back to `*`, which is fine, but naming them documents the intent and lets us
# tune crawl budget per engine. Keep the DISALLOW block identical to `*`.
#
# Baidu ignores hreflang entirely and judges a page by its own zh title /
# description / content-language, so /zh-CN has to stand on its own; letting
# Baiduspider through is the prerequisite for that.
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

# Baidu's JS-rendering crawler — needed because the homepage sections are React.
User-agent: Baiduspider-render
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

# Shenma (UC/Alibaba) — mobile search, meaningful share on Chinese phones.
User-agent: YisouSpider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Sogou web spider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: Sogou inst spider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: 360Spider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

User-agent: HaosouSpider
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

# Huawei Petal search (also feeds Huawei's assistant surfaces).
User-agent: PetalBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /blog/cms/

# Sitemap (canonical host is www — apex 301s to www, so list www directly)
Sitemap: https://www.rinda.ai/sitemap.xml
Sitemap: https://www.rinda.ai/video-sitemap.xml