chore: enhance robots.txt with search engine policies and portal exclusion

src/TaxBaik.Web/wwwroot/robots.txt

@@ -1,7 +1,33 @@
+# TaxBaik robots.txt
+# 공개 콘텐츠: 허용 (블로그, FAQ, 공지, 상담)
+# 관리 페이지: 차단 (admin, portal)
+
 User-agent: *
 Allow: /
 Disallow: /admin/
+Disallow: /portal/
 Disallow: /manage/
+Disallow: *.json$
+Disallow: *.xml$ (sitemap 제외)
 
+# 검색 엔진별 크롤링 속도 제한
+User-agent: Googlebot
+Crawl-delay: 1
+
+User-agent: Bingbot
+Crawl-delay: 2
+
+User-agent: Yeti
+Crawl-delay: 1
+
+# Naver, Kakao 검색 엔진
+User-agent: Naver
+Allow: /
+
+User-agent: Kakaobot
+Allow: /
+
+# Sitemap 위치
 Sitemap: https://www.taxbaik.com/taxbaik/sitemap.xml
 Sitemap: https://taxbaik.com/taxbaik/sitemap.xml
+Sitemap: https://www.taxbaik.com/taxbaik/robots.txt