diff --git a/src/TaxBaik.Web/wwwroot/robots.txt b/src/TaxBaik.Web/wwwroot/robots.txt index 6e39358..025ece5 100644 --- a/src/TaxBaik.Web/wwwroot/robots.txt +++ b/src/TaxBaik.Web/wwwroot/robots.txt @@ -1,7 +1,33 @@ +# TaxBaik robots.txt +# 공개 콘텐츠: 허용 (블로그, FAQ, 공지, 상담) +# 관리 페이지: 차단 (admin, portal) + User-agent: * Allow: / Disallow: /admin/ +Disallow: /portal/ Disallow: /manage/ +Disallow: *.json$ +Disallow: *.xml$ (sitemap 제외) +# 검색 엔진별 크롤링 속도 제한 +User-agent: Googlebot +Crawl-delay: 1 + +User-agent: Bingbot +Crawl-delay: 2 + +User-agent: Yeti +Crawl-delay: 1 + +# Naver, Kakao 검색 엔진 +User-agent: Naver +Allow: / + +User-agent: Kakaobot +Allow: / + +# Sitemap 위치 Sitemap: https://www.taxbaik.com/taxbaik/sitemap.xml Sitemap: https://taxbaik.com/taxbaik/sitemap.xml +Sitemap: https://www.taxbaik.com/taxbaik/robots.txt