섹터 유니버스 분리와 월간 갱신 정합화
This commit is contained in:
@@ -15,5 +15,5 @@
|
|||||||
"keep package scripts within release envelope"
|
"keep package scripts within release envelope"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"source_zip_sha256": "49f64b3773ba3c19fa8323d0b08833928c637935483039579bb8ab22a391f70c"
|
"source_zip_sha256": "4de4a7b1217ef5d5375b3b1ea1209f738719e79c4c3c0954e9e96a9dc0d8607e"
|
||||||
}
|
}
|
||||||
@@ -13,7 +13,7 @@ portfolio_exposure_framework:
|
|||||||
exposure_layers:
|
exposure_layers:
|
||||||
direct_core_leaders: ["삼성전자", "SK하이닉스"]
|
direct_core_leaders: ["삼성전자", "SK하이닉스"]
|
||||||
duplicate_beta: ["KODEX 반도체", "동일 섹터 ETF"]
|
duplicate_beta: ["KODEX 반도체", "동일 섹터 ETF"]
|
||||||
tactical_satellites: ["방산", "조선", "전력기기", "건설/EPC", "기타 고베타"]
|
tactical_satellites: ["방산", "조선", "전력설비", "건설", "플랜트/EPC", "로보틱스", "기타 고베타"]
|
||||||
cash: ["현금", "MMF", "RP", "단기채 ETF"]
|
cash: ["현금", "MMF", "RP", "단기채 ETF"]
|
||||||
valid_trim_reasons:
|
valid_trim_reasons:
|
||||||
- "벤치마크 대비 초과비중이 허용밴드를 초과하고 가격 추세가 훼손됨"
|
- "벤치마크 대비 초과비중이 허용밴드를 초과하고 가격 추세가 훼손됨"
|
||||||
|
|||||||
+402
-27
@@ -1,5 +1,5 @@
|
|||||||
// gas_lib.gs - Common utilities & static features
|
// gas_lib.gs - Common utilities & static features
|
||||||
// Last Updated: 2026-06-14 20:48:30 KST
|
// Last Updated: 2026-06-15 02:20:50 KST
|
||||||
// Math/KRX utils, sheet I/O, sector flow, Web API, static runners
|
// Math/KRX utils, sheet I/O, sector flow, Web API, static runners
|
||||||
// GAS global scope: functions in gas_data_feed.gs / gas_data_collect.gs callable directly
|
// GAS global scope: functions in gas_data_feed.gs / gas_data_collect.gs callable directly
|
||||||
//
|
//
|
||||||
@@ -593,7 +593,14 @@ const DEFAULT_SECTOR_UNIVERSE_V2 = [
|
|||||||
{ code: "062040", name: "산일전기", weight: 0.10 },
|
{ code: "062040", name: "산일전기", weight: 0.10 },
|
||||||
{ code: "298040", name: "효성중공업", weight: 0.10 },
|
{ code: "298040", name: "효성중공업", weight: 0.10 },
|
||||||
]},
|
]},
|
||||||
{ sector: "방산", proxyTicker: "012450", proxyName: "한화에어로스페이스", proxyType: "대표주", baseTicker: "069500", constituents: [
|
{ sector: "전력설비", proxyTicker: "491820", proxyName: "HANARO 전력설비투자", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
|
{ code: "010120", name: "LS ELECTRIC", weight: 0.28 },
|
||||||
|
{ code: "267260", name: "HD현대일렉트릭", weight: 0.28 },
|
||||||
|
{ code: "298040", name: "효성중공업", weight: 0.18 },
|
||||||
|
{ code: "006260", name: "LS", weight: 0.14 },
|
||||||
|
{ code: "099440", name: "두산에너빌리티", weight: 0.12 },
|
||||||
|
]},
|
||||||
|
{ sector: "방산", proxyTicker: "463250", proxyName: "TIGER K방산&우주", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "012450", name: "한화에어로스페이스", weight: 0.45 },
|
{ code: "012450", name: "한화에어로스페이스", weight: 0.45 },
|
||||||
{ code: "079550", name: "LIG넥스원", weight: 0.25 },
|
{ code: "079550", name: "LIG넥스원", weight: 0.25 },
|
||||||
{ code: "047810", name: "한국항공우주", weight: 0.15 },
|
{ code: "047810", name: "한국항공우주", weight: 0.15 },
|
||||||
@@ -605,23 +612,49 @@ const DEFAULT_SECTOR_UNIVERSE_V2 = [
|
|||||||
{ code: "009540", name: "HD한국조선해양", weight: 0.20 },
|
{ code: "009540", name: "HD한국조선해양", weight: 0.20 },
|
||||||
{ code: "494670", name: "TIGER 조선TOP10", weight: 0.15, isEtf: true },
|
{ code: "494670", name: "TIGER 조선TOP10", weight: 0.15, isEtf: true },
|
||||||
]},
|
]},
|
||||||
{ sector: "건설/EPC", proxyTicker: "028050", proxyName: "삼성E&A", proxyType: "대표주", baseTicker: "069500", constituents: [
|
{ sector: "건설", proxyTicker: "117700", proxyName: "KODEX 건설", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "028050", name: "삼성E&A", weight: 0.40 },
|
{ code: "000720", name: "현대건설", weight: 0.35 },
|
||||||
{ code: "000720", name: "현대건설", weight: 0.30 },
|
{ code: "006360", name: "GS건설", weight: 0.25 },
|
||||||
{ code: "006360", name: "GS건설", weight: 0.20 },
|
{ code: "047040", name: "대우건설", weight: 0.20 },
|
||||||
{ code: "047040", name: "대우건설", weight: 0.10 },
|
{ code: "294870", name: "HDC현대산업개발", weight: 0.20 },
|
||||||
|
]},
|
||||||
|
{ sector: "플랜트/EPC", proxyTicker: "454320", proxyName: "HANARO CAPEX설비투자iSelect", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
|
{ code: "028050", name: "삼성E&A", weight: 0.35 },
|
||||||
|
{ code: "010120", name: "LS ELECTRIC", weight: 0.20 },
|
||||||
|
{ code: "267260", name: "HD현대일렉트릭", weight: 0.20 },
|
||||||
|
{ code: "298040", name: "효성중공업", weight: 0.15 },
|
||||||
|
{ code: "099440", name: "두산에너빌리티", weight: 0.10 },
|
||||||
]},
|
]},
|
||||||
{ sector: "자동차", proxyTicker: "091180", proxyName: "TIGER 자동차", proxyType: "ETF", baseTicker: "069500", constituents: [
|
{ sector: "자동차", proxyTicker: "091180", proxyName: "TIGER 자동차", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "005380", name: "현대차", weight: 0.45 },
|
{ code: "005380", name: "현대차", weight: 0.45 },
|
||||||
{ code: "000270", name: "기아", weight: 0.40 },
|
{ code: "000270", name: "기아", weight: 0.40 },
|
||||||
{ code: "012330", name: "현대모비스", weight: 0.15 },
|
{ code: "012330", name: "현대모비스", weight: 0.15 },
|
||||||
]},
|
]},
|
||||||
{ sector: "금융/은행", proxyTicker: "091170", proxyName: "KODEX 은행", proxyType: "ETF", baseTicker: "069500", constituents: [
|
{ sector: "은행", proxyTicker: "091170", proxyName: "KODEX 은행", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "105560", name: "KB금융", weight: 0.30 },
|
{ code: "105560", name: "KB금융", weight: 0.30 },
|
||||||
{ code: "055550", name: "신한지주", weight: 0.30 },
|
{ code: "055550", name: "신한지주", weight: 0.30 },
|
||||||
{ code: "086790", name: "하나금융지주", weight: 0.20 },
|
{ code: "086790", name: "하나금융지주", weight: 0.20 },
|
||||||
{ code: "316140", name: "우리금융지주", weight: 0.10 },
|
{ code: "316140", name: "우리금융지주", weight: 0.10 },
|
||||||
{ code: "003540", name: "대신증권", weight: 0.10 },
|
{ code: "024110", name: "기업은행", weight: 0.10 },
|
||||||
|
]},
|
||||||
|
{ sector: "증권", proxyTicker: "0111J0", proxyName: "HANARO 증권고배당TOP3플러스", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
|
{ code: "071050", name: "한국금융지주", weight: 0.2135 },
|
||||||
|
{ code: "006800", name: "미래에셋증권", weight: 0.1934 },
|
||||||
|
{ code: "005940", name: "NH투자증권", weight: 0.1911 },
|
||||||
|
{ code: "016360", name: "삼성증권", weight: 0.1434 },
|
||||||
|
{ code: "039490", name: "키움증권", weight: 0.1373 },
|
||||||
|
]},
|
||||||
|
{ sector: "지주회사", proxyTicker: "307520", proxyName: "TIGER 지주회사", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
|
{ code: "180640", name: "한진칼", weight: 0.1535 },
|
||||||
|
{ code: "267250", name: "HD현대", weight: 0.0943 },
|
||||||
|
{ code: "034730", name: "SK", weight: 0.0884 },
|
||||||
|
{ code: "000150", name: "두산", weight: 0.0878 },
|
||||||
|
{ code: "005490", name: "POSCO홀딩스", weight: 0.0763 },
|
||||||
|
{ code: "003550", name: "LG", weight: 0.0752 },
|
||||||
|
{ code: "006260", name: "LS", weight: 0.0705 },
|
||||||
|
{ code: "078930", name: "GS", weight: 0.0498 },
|
||||||
|
{ code: "001040", name: "CJ", weight: 0.0477 },
|
||||||
|
{ code: "010060", name: "OCI홀딩스", weight: 0.0240 },
|
||||||
]},
|
]},
|
||||||
{ sector: "2차전지", proxyTicker: "305720", proxyName: "KODEX 2차전지산업", proxyType: "ETF", baseTicker: "069500", constituents: [
|
{ sector: "2차전지", proxyTicker: "305720", proxyName: "KODEX 2차전지산업", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "373220", name: "LG에너지솔루션", weight: 0.40 },
|
{ code: "373220", name: "LG에너지솔루션", weight: 0.40 },
|
||||||
@@ -635,12 +668,29 @@ const DEFAULT_SECTOR_UNIVERSE_V2 = [
|
|||||||
{ code: "128940", name: "한미약품", weight: 0.15 },
|
{ code: "128940", name: "한미약품", weight: 0.15 },
|
||||||
{ code: "000100", name: "유한양행", weight: 0.10 },
|
{ code: "000100", name: "유한양행", weight: 0.10 },
|
||||||
]},
|
]},
|
||||||
{ sector: "원전", proxyTicker: "099440", proxyName: "두산에너빌리티", proxyType: "대표주", baseTicker: "069500", constituents: [
|
{ sector: "원전", proxyTicker: "434730", proxyName: "HANARO 원자력iSelect", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "099440", name: "두산에너빌리티", weight: 0.45 },
|
{ code: "099440", name: "두산에너빌리티", weight: 0.45 },
|
||||||
{ code: "023450", name: "한전기술", weight: 0.25 },
|
{ code: "023450", name: "한전기술", weight: 0.25 },
|
||||||
{ code: "015760", name: "한국전력", weight: 0.20 },
|
{ code: "015760", name: "한국전력", weight: 0.20 },
|
||||||
{ code: "071320", name: "지역난방공사", weight: 0.10 },
|
{ code: "071320", name: "지역난방공사", weight: 0.10 },
|
||||||
]},
|
]},
|
||||||
|
{ sector: "로보틱스", proxyTicker: "0190C0", proxyName: "RISE 현대차고정피지컬AI", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
|
{ code: "005380", name: "현대차", weight: 0.2402 },
|
||||||
|
{ code: "012330", name: "현대모비스", weight: 0.1588 },
|
||||||
|
{ code: "011070", name: "LG이노텍", weight: 0.1450 },
|
||||||
|
{ code: "000270", name: "기아", weight: 0.1234 },
|
||||||
|
{ code: "307950", name: "현대오토에버", weight: 0.0899 },
|
||||||
|
{ code: "277810", name: "레인보우로보틱스", weight: 0.0673 },
|
||||||
|
{ code: "064400", name: "LG씨엔에스", weight: 0.0519 },
|
||||||
|
{ code: "454910", name: "두산로보틱스", weight: 0.0367 },
|
||||||
|
{ code: "108490", name: "로보티즈", weight: 0.0240 },
|
||||||
|
{ code: "058610", name: "에스피지", weight: 0.0173 },
|
||||||
|
{ code: "010620", name: "현대미포", weight: 0.0135 },
|
||||||
|
{ code: "009540", name: "HD한국조선해양", weight: 0.0135 },
|
||||||
|
{ code: "011210", name: "현대위아", weight: 0.0109 },
|
||||||
|
{ code: "121600", name: "나노신소재", weight: 0.0040 },
|
||||||
|
{ code: "028050", name: "삼성E&A", weight: 0.0034 },
|
||||||
|
]},
|
||||||
{ sector: "소비재", proxyTicker: "139220", proxyName: "TIGER 생활소비재", proxyType: "ETF", baseTicker: "069500", constituents: [
|
{ sector: "소비재", proxyTicker: "139220", proxyName: "TIGER 생활소비재", proxyType: "ETF", baseTicker: "069500", constituents: [
|
||||||
{ code: "028260", name: "삼성물산", weight: 0.35 },
|
{ code: "028260", name: "삼성물산", weight: 0.35 },
|
||||||
{ code: "097950", name: "CJ제일제당", weight: 0.25 },
|
{ code: "097950", name: "CJ제일제당", weight: 0.25 },
|
||||||
@@ -663,6 +713,7 @@ function normalizeSectorName_(sector) {
|
|||||||
if (s === "바이오/헬스케어") return "바이오";
|
if (s === "바이오/헬스케어") return "바이오";
|
||||||
if (s === "원전/에너지") return "원전";
|
if (s === "원전/에너지") return "원전";
|
||||||
if (s === "소비재/유통") return "소비재";
|
if (s === "소비재/유통") return "소비재";
|
||||||
|
if (s === "건설/EPC") return "플랜트/EPC";
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -679,17 +730,52 @@ function readSectorUniverse_() {
|
|||||||
const sheet = ss.getSheetByName("sector_universe");
|
const sheet = ss.getSheetByName("sector_universe");
|
||||||
if (!sheet) {
|
if (!sheet) {
|
||||||
writeDefaultSectorUniverseSheet_();
|
writeDefaultSectorUniverseSheet_();
|
||||||
return DEFAULT_SECTOR_UNIVERSE_V2;
|
return DEFAULT_SECTOR_UNIVERSE_V2.map(sector => ({
|
||||||
|
...sector,
|
||||||
|
source: sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
sourceUrl: sector.sourceUrl || "",
|
||||||
|
sourceAsOf: sector.sourceAsOf || "",
|
||||||
|
constituents: sector.constituents.map(c => ({
|
||||||
|
...c,
|
||||||
|
source: c.source || sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
sourceUrl: c.sourceUrl || sector.sourceUrl || "",
|
||||||
|
sourceAsOf: c.sourceAsOf || sector.sourceAsOf || "",
|
||||||
|
})),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
const data = sheet.getDataRange().getValues();
|
const data = sheet.getDataRange().getValues();
|
||||||
if (data.length < 3) {
|
if (data.length < 3) {
|
||||||
writeDefaultSectorUniverseSheet_();
|
writeDefaultSectorUniverseSheet_();
|
||||||
return DEFAULT_SECTOR_UNIVERSE_V2;
|
return DEFAULT_SECTOR_UNIVERSE_V2.map(sector => ({
|
||||||
|
...sector,
|
||||||
|
source: sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
sourceUrl: sector.sourceUrl || "",
|
||||||
|
sourceAsOf: sector.sourceAsOf || "",
|
||||||
|
constituents: sector.constituents.map(c => ({
|
||||||
|
...c,
|
||||||
|
source: c.source || sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
sourceUrl: c.sourceUrl || sector.sourceUrl || "",
|
||||||
|
sourceAsOf: c.sourceAsOf || sector.sourceAsOf || "",
|
||||||
|
})),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
const hdr = data[1].map(h => String(h).trim());
|
const hdr = data[1].map(h => String(h).trim());
|
||||||
const idx = name => hdr.indexOf(name);
|
const idx = name => hdr.indexOf(name);
|
||||||
const required = ["Sector","Proxy_Ticker","Constituent_Code","Weight"];
|
const required = ["Sector","Proxy_Ticker","Constituent_Code","Weight"];
|
||||||
if (required.some(h => idx(h) < 0)) return DEFAULT_SECTOR_UNIVERSE_V2;
|
if (required.some(h => idx(h) < 0)) {
|
||||||
|
return DEFAULT_SECTOR_UNIVERSE_V2.map(sector => ({
|
||||||
|
...sector,
|
||||||
|
source: sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
sourceUrl: sector.sourceUrl || "",
|
||||||
|
sourceAsOf: sector.sourceAsOf || "",
|
||||||
|
constituents: sector.constituents.map(c => ({
|
||||||
|
...c,
|
||||||
|
source: c.source || sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
sourceUrl: c.sourceUrl || sector.sourceUrl || "",
|
||||||
|
sourceAsOf: c.sourceAsOf || sector.sourceAsOf || "",
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
const map = {};
|
const map = {};
|
||||||
for (let i = 2; i < data.length; i++) {
|
for (let i = 2; i < data.length; i++) {
|
||||||
@@ -706,6 +792,9 @@ function readSectorUniverse_() {
|
|||||||
proxyName: idx("Proxy_Name") >= 0 ? String(data[i][idx("Proxy_Name")] ?? "").trim() : "",
|
proxyName: idx("Proxy_Name") >= 0 ? String(data[i][idx("Proxy_Name")] ?? "").trim() : "",
|
||||||
proxyType: idx("Proxy_Type") >= 0 ? String(data[i][idx("Proxy_Type")] ?? "").trim() : "",
|
proxyType: idx("Proxy_Type") >= 0 ? String(data[i][idx("Proxy_Type")] ?? "").trim() : "",
|
||||||
baseTicker: idx("Base_Ticker") >= 0 ? normalizeTickerCode(data[i][idx("Base_Ticker")]) : "069500",
|
baseTicker: idx("Base_Ticker") >= 0 ? normalizeTickerCode(data[i][idx("Base_Ticker")]) : "069500",
|
||||||
|
source: idx("Source") >= 0 ? String(data[i][idx("Source")] ?? "").trim() : "SHEET_INPUT",
|
||||||
|
sourceUrl: idx("Source_URL") >= 0 ? String(data[i][idx("Source_URL")] ?? "").trim() : "",
|
||||||
|
sourceAsOf: idx("Source_AsOf") >= 0 ? String(data[i][idx("Source_AsOf")] ?? "").trim() : "",
|
||||||
constituents: [],
|
constituents: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -714,16 +803,59 @@ function readSectorUniverse_() {
|
|||||||
name: idx("Constituent_Name") >= 0 ? String(data[i][idx("Constituent_Name")] ?? "").trim() : "",
|
name: idx("Constituent_Name") >= 0 ? String(data[i][idx("Constituent_Name")] ?? "").trim() : "",
|
||||||
weight,
|
weight,
|
||||||
isEtf: idx("Is_ETF") >= 0 ? boolFromSheet_(data[i][idx("Is_ETF")], false) : false,
|
isEtf: idx("Is_ETF") >= 0 ? boolFromSheet_(data[i][idx("Is_ETF")], false) : false,
|
||||||
|
source: idx("Source") >= 0 ? String(data[i][idx("Source")] ?? "").trim() : "SHEET_INPUT",
|
||||||
|
transportMode: idx("Transport_Mode") >= 0 ? String(data[i][idx("Transport_Mode")] ?? "").trim() : "",
|
||||||
|
sourceUrl: idx("Source_URL") >= 0 ? String(data[i][idx("Source_URL")] ?? "").trim() : "",
|
||||||
|
sourceAsOf: idx("Source_AsOf") >= 0 ? String(data[i][idx("Source_AsOf")] ?? "").trim() : "",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const sectors = Object.values(map).filter(s => s.proxyTicker && s.constituents.length > 0);
|
const sectors = Object.values(map).filter(s => s.proxyTicker && s.constituents.length > 0);
|
||||||
return sectors.length ? sectors : DEFAULT_SECTOR_UNIVERSE_V2;
|
const sectorSet = new Set(sectors.map(s => s.sector));
|
||||||
|
for (const fallback of DEFAULT_SECTOR_UNIVERSE_V2) {
|
||||||
|
if (!fallback || !fallback.sector || sectorSet.has(fallback.sector)) continue;
|
||||||
|
sectors.push({
|
||||||
|
sector: fallback.sector,
|
||||||
|
proxyTicker: fallback.proxyTicker,
|
||||||
|
proxyName: fallback.proxyName,
|
||||||
|
proxyType: fallback.proxyType,
|
||||||
|
baseTicker: fallback.baseTicker || "069500",
|
||||||
|
source: fallback.source || "DEFAULT_TEMPLATE",
|
||||||
|
transportMode: fallback.transportMode || ((fallback.source || "DEFAULT_TEMPLATE") === "NAVER_ETF_PAGE" || (fallback.source || "DEFAULT_TEMPLATE") === "REPRESENTATIVE_STOCK_PROXY" ? "HTML_SERVER_RENDERED" : "MANUAL_OR_TEMPLATE"),
|
||||||
|
sourceUrl: fallback.sourceUrl || "",
|
||||||
|
sourceAsOf: fallback.sourceAsOf || "",
|
||||||
|
constituents: fallback.constituents.map(c => ({
|
||||||
|
code: c.code,
|
||||||
|
name: c.name || "",
|
||||||
|
weight: c.weight,
|
||||||
|
isEtf: Boolean(c.isEtf),
|
||||||
|
source: c.source || fallback.source || "DEFAULT_TEMPLATE",
|
||||||
|
transportMode: c.transportMode || ((c.source || fallback.source || "DEFAULT_TEMPLATE") === "NAVER_ETF_PAGE" || (c.source || fallback.source || "DEFAULT_TEMPLATE") === "REPRESENTATIVE_STOCK_PROXY" ? "HTML_SERVER_RENDERED" : "MANUAL_OR_TEMPLATE"),
|
||||||
|
sourceUrl: c.sourceUrl || fallback.sourceUrl || "",
|
||||||
|
sourceAsOf: c.sourceAsOf || fallback.sourceAsOf || "",
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return sectors.length ? sectors : DEFAULT_SECTOR_UNIVERSE_V2.map(sector => ({
|
||||||
|
...sector,
|
||||||
|
source: sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
transportMode: sector.transportMode || ((sector.source || "DEFAULT_TEMPLATE") === "NAVER_ETF_PAGE" || (sector.source || "DEFAULT_TEMPLATE") === "REPRESENTATIVE_STOCK_PROXY" ? "HTML_SERVER_RENDERED" : "MANUAL_OR_TEMPLATE"),
|
||||||
|
sourceUrl: sector.sourceUrl || "",
|
||||||
|
sourceAsOf: sector.sourceAsOf || "",
|
||||||
|
constituents: sector.constituents.map(c => ({
|
||||||
|
...c,
|
||||||
|
source: c.source || sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
transportMode: c.transportMode || ((c.source || sector.source || "DEFAULT_TEMPLATE") === "NAVER_ETF_PAGE" || (c.source || sector.source || "DEFAULT_TEMPLATE") === "REPRESENTATIVE_STOCK_PROXY" ? "HTML_SERVER_RENDERED" : "MANUAL_OR_TEMPLATE"),
|
||||||
|
sourceUrl: c.sourceUrl || sector.sourceUrl || "",
|
||||||
|
sourceAsOf: c.sourceAsOf || sector.sourceAsOf || "",
|
||||||
|
})),
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function writeDefaultSectorUniverseSheet_() {
|
function writeDefaultSectorUniverseSheet_() {
|
||||||
const headers = [
|
const headers = [
|
||||||
"Sector","Proxy_Ticker","Proxy_Name","Proxy_Type","Base_Ticker",
|
"Sector","Proxy_Ticker","Proxy_Name","Proxy_Type","Base_Ticker",
|
||||||
"Constituent_Code","Constituent_Name","Weight","Is_ETF","Enabled","Effective_Date","Source"
|
"Constituent_Code","Constituent_Name","Weight","Is_ETF","Enabled","Effective_Date","Source","Transport_Mode",
|
||||||
|
"Source_URL","Source_AsOf"
|
||||||
];
|
];
|
||||||
const today = Utilities.formatDate(new Date(), "Asia/Seoul", "yyyy-MM-dd");
|
const today = Utilities.formatDate(new Date(), "Asia/Seoul", "yyyy-MM-dd");
|
||||||
const rows = [];
|
const rows = [];
|
||||||
@@ -741,7 +873,10 @@ function writeDefaultSectorUniverseSheet_() {
|
|||||||
c.isEtf ? "Y" : "N",
|
c.isEtf ? "Y" : "N",
|
||||||
"Y",
|
"Y",
|
||||||
today,
|
today,
|
||||||
"sector_universe(DEFAULT_SECTOR_UNIVERSE_V2)",
|
sector.source || c.source || "DEFAULT_TEMPLATE",
|
||||||
|
sector.transportMode || c.transportMode || (((sector.source || c.source || "DEFAULT_TEMPLATE") === "NAVER_ETF_PAGE" || (sector.source || c.source || "DEFAULT_TEMPLATE") === "REPRESENTATIVE_STOCK_PROXY") ? "HTML_SERVER_RENDERED" : "MANUAL_OR_TEMPLATE"),
|
||||||
|
sector.sourceUrl || c.sourceUrl || "",
|
||||||
|
sector.sourceAsOf || c.sourceAsOf || "",
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -762,6 +897,228 @@ function sectorUseMode_(quality) {
|
|||||||
return "INVALID";
|
return "INVALID";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseDateOnly_(value) {
|
||||||
|
const text = String(value ?? "").trim();
|
||||||
|
if (!text) return null;
|
||||||
|
const norm = text.replace(/\./g, "-").slice(0, 10);
|
||||||
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(norm)) return null;
|
||||||
|
const parsed = new Date(norm + "T00:00:00+09:00");
|
||||||
|
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function calcSectorUniverseRefreshAudit_(universe) {
|
||||||
|
const today = new Date();
|
||||||
|
const rows = [];
|
||||||
|
const sourceKindCounts = { NAVER_ETF_PAGE: 0, NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED: 0, NAVER_ETF_PAGE_FAIL: 0, REPRESENTATIVE_STOCK_PROXY: 0, SHEET_INPUT: 0, DEFAULT_TEMPLATE: 0, OTHER: 0 };
|
||||||
|
const transportModeCounts = { HTML_SERVER_RENDERED: 0, MANUAL_OR_TEMPLATE: 0, LAYOUT_CHANGED: 0, UNKNOWN: 0 };
|
||||||
|
let currentCount = 0;
|
||||||
|
let dueCount = 0;
|
||||||
|
let overdueCount = 0;
|
||||||
|
let missingCount = 0;
|
||||||
|
let templateCount = 0;
|
||||||
|
let sheetInputCount = 0;
|
||||||
|
let naverSourceCount = 0;
|
||||||
|
let layoutChangedCount = 0;
|
||||||
|
let missingSourceUrlCount = 0;
|
||||||
|
let staleSectorCount = 0;
|
||||||
|
let oldestSourceAsOf = null;
|
||||||
|
let newestSourceAsOf = null;
|
||||||
|
|
||||||
|
for (const sector of universe || []) {
|
||||||
|
const sectorRows = Array.isArray(sector?.constituents) ? sector.constituents : [];
|
||||||
|
const sourceKind = String(sector?.source || "SHEET_INPUT").trim() || "SHEET_INPUT";
|
||||||
|
if (Object.prototype.hasOwnProperty.call(sourceKindCounts, sourceKind)) {
|
||||||
|
sourceKindCounts[sourceKind] += 1;
|
||||||
|
} else {
|
||||||
|
sourceKindCounts.OTHER += 1;
|
||||||
|
}
|
||||||
|
const transportMode = String(sector?.transportMode || "").trim() ||
|
||||||
|
(sourceKind === "NAVER_ETF_PAGE" || sourceKind === "REPRESENTATIVE_STOCK_PROXY" ? "HTML_SERVER_RENDERED" :
|
||||||
|
sourceKind === "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" ? "LAYOUT_CHANGED" :
|
||||||
|
(sourceKind === "DEFAULT_TEMPLATE" || sourceKind === "SHEET_INPUT" ? "MANUAL_OR_TEMPLATE" : "UNKNOWN"));
|
||||||
|
if (Object.prototype.hasOwnProperty.call(transportModeCounts, transportMode)) {
|
||||||
|
transportModeCounts[transportMode] += 1;
|
||||||
|
} else {
|
||||||
|
transportModeCounts.UNKNOWN += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sourceUrl = String(sector?.sourceUrl || "").trim();
|
||||||
|
const sourceAsOf = String(sector?.sourceAsOf || "").trim();
|
||||||
|
const parsed = parseDateOnly_(sourceAsOf);
|
||||||
|
const ageDays = parsed ? Math.floor((today.getTime() - parsed.getTime()) / 86400000) : null;
|
||||||
|
if (parsed) {
|
||||||
|
oldestSourceAsOf = oldestSourceAsOf && oldestSourceAsOf < parsed ? oldestSourceAsOf : parsed;
|
||||||
|
newestSourceAsOf = newestSourceAsOf && newestSourceAsOf > parsed ? newestSourceAsOf : parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
let status = "INVALID";
|
||||||
|
const reasons = [];
|
||||||
|
if (sourceKind === "DEFAULT_TEMPLATE") {
|
||||||
|
status = "TEMPLATE";
|
||||||
|
templateCount += 1;
|
||||||
|
reasons.push("DEFAULT_TEMPLATE");
|
||||||
|
} else if (sourceKind === "REPRESENTATIVE_STOCK_PROXY") {
|
||||||
|
if (!sourceUrl) {
|
||||||
|
status = "MISSING";
|
||||||
|
missingCount += 1;
|
||||||
|
missingSourceUrlCount += 1;
|
||||||
|
reasons.push("Source_URL_MISSING");
|
||||||
|
} else if (ageDays === null) {
|
||||||
|
status = "MISSING";
|
||||||
|
missingCount += 1;
|
||||||
|
reasons.push("Source_AsOf_MISSING");
|
||||||
|
} else if (ageDays <= 31) {
|
||||||
|
status = "CURRENT";
|
||||||
|
currentCount += 1;
|
||||||
|
} else if (ageDays <= 45) {
|
||||||
|
status = "DUE";
|
||||||
|
dueCount += 1;
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
} else {
|
||||||
|
status = "OVERDUE";
|
||||||
|
overdueCount += 1;
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
}
|
||||||
|
} else if (sourceKind === "SHEET_INPUT") {
|
||||||
|
sheetInputCount += 1;
|
||||||
|
if (!sourceUrl) {
|
||||||
|
status = "MISSING";
|
||||||
|
missingCount += 1;
|
||||||
|
missingSourceUrlCount += 1;
|
||||||
|
reasons.push("Source_URL_MISSING");
|
||||||
|
} else if (ageDays === null) {
|
||||||
|
status = "MISSING";
|
||||||
|
missingCount += 1;
|
||||||
|
reasons.push("Source_AsOf_MISSING");
|
||||||
|
} else if (ageDays <= 31) {
|
||||||
|
status = "CURRENT";
|
||||||
|
currentCount += 1;
|
||||||
|
} else if (ageDays <= 45) {
|
||||||
|
status = "DUE";
|
||||||
|
dueCount += 1;
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
} else {
|
||||||
|
status = "OVERDUE";
|
||||||
|
overdueCount += 1;
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
}
|
||||||
|
} else if (sourceKind === "NAVER_ETF_PAGE") {
|
||||||
|
naverSourceCount += 1;
|
||||||
|
if (!sourceUrl) {
|
||||||
|
status = "MISSING";
|
||||||
|
missingCount += 1;
|
||||||
|
missingSourceUrlCount += 1;
|
||||||
|
reasons.push("Source_URL_MISSING");
|
||||||
|
} else if (ageDays === null) {
|
||||||
|
status = "MISSING";
|
||||||
|
missingCount += 1;
|
||||||
|
reasons.push("Source_AsOf_MISSING");
|
||||||
|
} else if (ageDays <= 31) {
|
||||||
|
status = "CURRENT";
|
||||||
|
currentCount += 1;
|
||||||
|
} else if (ageDays <= 45) {
|
||||||
|
status = "DUE";
|
||||||
|
dueCount += 1;
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
} else {
|
||||||
|
status = "OVERDUE";
|
||||||
|
overdueCount += 1;
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
}
|
||||||
|
} else if (sourceKind === "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED") {
|
||||||
|
layoutChangedCount += 1;
|
||||||
|
status = "LAYOUT_CHANGED";
|
||||||
|
if (!sourceUrl) {
|
||||||
|
missingSourceUrlCount += 1;
|
||||||
|
reasons.push("Source_URL_MISSING");
|
||||||
|
}
|
||||||
|
if (ageDays === null) {
|
||||||
|
reasons.push("Source_AsOf_MISSING");
|
||||||
|
} else {
|
||||||
|
staleSectorCount += 1;
|
||||||
|
reasons.push(`AgeDays=${ageDays}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
status = "INVALID";
|
||||||
|
reasons.push("SOURCE_KIND_UNKNOWN");
|
||||||
|
if (!sourceUrl) missingSourceUrlCount += 1;
|
||||||
|
}
|
||||||
|
if (!sourceUrl) reasons.push("Source_URL_MISSING");
|
||||||
|
if (ageDays !== null && ageDays < 0) reasons.push("FUTURE_DATE");
|
||||||
|
|
||||||
|
rows.push({
|
||||||
|
sector: sector.sector || "",
|
||||||
|
proxy_ticker: sector.proxyTicker || "",
|
||||||
|
proxy_name: sector.proxyName || "",
|
||||||
|
proxy_type: sector.proxyType || "",
|
||||||
|
source_kind: sourceKind,
|
||||||
|
transport_mode: transportMode,
|
||||||
|
source_url: sourceUrl,
|
||||||
|
source_asof: sourceAsOf,
|
||||||
|
age_days: ageDays === null ? "" : ageDays,
|
||||||
|
constituent_count: sectorRows.length,
|
||||||
|
stock_count: sectorRows.filter(c => !c.isEtf).length,
|
||||||
|
etf_count: sectorRows.filter(c => c.isEtf).length,
|
||||||
|
weight_sum: sectorRows.reduce((a, c) => a + (Number(c.weight) || 0), 0),
|
||||||
|
status: status,
|
||||||
|
refresh_reason: reasons.length ? reasons.join(";") : "OK",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
rows.sort((a, b) => {
|
||||||
|
if (a.status === "CURRENT" && b.status !== "CURRENT") return -1;
|
||||||
|
if (a.status !== "CURRENT" && b.status === "CURRENT") return 1;
|
||||||
|
return String(a.sector || "").localeCompare(String(b.sector || ""));
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
formula_id: "sector_universe_refresh_audit_v1",
|
||||||
|
gate: (templateCount > 0 || missingSourceUrlCount > 0 || overdueCount > 0 || staleSectorCount > 0) ? "FAIL" : (sheetInputCount > 0 ? "WARN" : "PASS"),
|
||||||
|
summary: {
|
||||||
|
sector_count: (universe || []).length,
|
||||||
|
current_count: currentCount,
|
||||||
|
due_count: dueCount,
|
||||||
|
overdue_count: overdueCount,
|
||||||
|
missing_count: missingCount,
|
||||||
|
template_count: templateCount,
|
||||||
|
sheet_input_count: sheetInputCount,
|
||||||
|
naver_source_count: naverSourceCount,
|
||||||
|
layout_changed_count: layoutChangedCount,
|
||||||
|
missing_source_url_count: missingSourceUrlCount,
|
||||||
|
stale_sector_count: staleSectorCount,
|
||||||
|
oldest_source_asof: oldestSourceAsOf ? Utilities.formatDate(oldestSourceAsOf, "Asia/Seoul", "yyyy-MM-dd") : "",
|
||||||
|
newest_source_asof: newestSourceAsOf ? Utilities.formatDate(newestSourceAsOf, "Asia/Seoul", "yyyy-MM-dd") : "",
|
||||||
|
source_kind_counts: sourceKindCounts,
|
||||||
|
transport_mode_counts: transportModeCounts,
|
||||||
|
ajax_mode: "NO",
|
||||||
|
transport_model: "HTML_SERVER_RENDERED",
|
||||||
|
},
|
||||||
|
rows: rows,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeSectorUniverseRefreshAuditSheet_(audit) {
|
||||||
|
if (!audit || typeof audit !== "object") return 0;
|
||||||
|
const headers = [
|
||||||
|
"sector", "proxy_ticker", "proxy_name", "proxy_type", "source_kind", "transport_mode",
|
||||||
|
"source_url", "source_asof", "age_days", "constituent_count",
|
||||||
|
"stock_count", "etf_count", "weight_sum", "status", "refresh_reason",
|
||||||
|
];
|
||||||
|
const rows = Array.isArray(audit.rows)
|
||||||
|
? audit.rows.map(function(r) {
|
||||||
|
return headers.map(function(h) { return r[h] ?? ""; });
|
||||||
|
})
|
||||||
|
: [];
|
||||||
|
writeToSheet("sector_universe_refresh_audit", headers, rows);
|
||||||
|
return rows.length;
|
||||||
|
}
|
||||||
|
|
||||||
function scoreSmartMoneyNorm_(v) {
|
function scoreSmartMoneyNorm_(v) {
|
||||||
if (!Number.isFinite(v)) return 0;
|
if (!Number.isFinite(v)) return 0;
|
||||||
if (v >= 0.15) return 25;
|
if (v >= 0.15) return 25;
|
||||||
@@ -955,7 +1312,7 @@ function runSectorFlowV3() {
|
|||||||
const etfRawMap = buildEtfRawMap_(buildEtfRawRows_(universe));
|
const etfRawMap = buildEtfRawMap_(buildEtfRawRows_(universe));
|
||||||
const today = Utilities.formatDate(new Date(), "Asia/Seoul", "yyyy-MM-dd");
|
const today = Utilities.formatDate(new Date(), "Asia/Seoul", "yyyy-MM-dd");
|
||||||
const headers = [
|
const headers = [
|
||||||
"Sector","Proxy_Ticker","Proxy_Name","Proxy_Type","Coverage_Weight",
|
"Sector","Proxy_Ticker","Proxy_Name","Proxy_Type","Universe_Source","Transport_Mode","Coverage_Weight",
|
||||||
"Sector_Ret5D","Sector_Ret20D","Sector_RS_20D",
|
"Sector_Ret5D","Sector_Ret20D","Sector_RS_20D",
|
||||||
"SmartMoney_5D_KRW","SmartMoney_20D_KRW","Sector_AvgTradeValue_20D_KRW","SmartMoney_5D_Norm",
|
"SmartMoney_5D_KRW","SmartMoney_20D_KRW","Sector_AvgTradeValue_20D_KRW","SmartMoney_5D_Norm",
|
||||||
"Flow_Breadth_5D","Flow_Rows_Min","Stale_Count",
|
"Flow_Breadth_5D","Flow_Rows_Min","Stale_Count",
|
||||||
@@ -1031,6 +1388,9 @@ function runSectorFlowV3() {
|
|||||||
const etfNavRisk = sector.proxyType === "ETF" ? (etfRaw?.navRisk ?? "NAV_DATA_MISSING") : "NOT_ETF";
|
const etfNavRisk = sector.proxyType === "ETF" ? (etfRaw?.navRisk ?? "NAV_DATA_MISSING") : "NOT_ETF";
|
||||||
const etfLiquidityStatus = sector.proxyType === "ETF" ? (etfRaw?.liquidityStatus ?? "WARN") : "NOT_ETF";
|
const etfLiquidityStatus = sector.proxyType === "ETF" ? (etfRaw?.liquidityStatus ?? "WARN") : "NOT_ETF";
|
||||||
const etfExecutionUse = sector.proxyType === "ETF" ? (etfRaw?.executionUse ?? "WATCH_ONLY") : "NOT_ETF";
|
const etfExecutionUse = sector.proxyType === "ETF" ? (etfRaw?.executionUse ?? "WATCH_ONLY") : "NOT_ETF";
|
||||||
|
const transportMode = sector.source === "NAVER_ETF_PAGE" ? "HTML_SERVER_RENDERED"
|
||||||
|
: (sector.source === "REPRESENTATIVE_STOCK_PROXY" ? "HTML_SERVER_RENDERED"
|
||||||
|
: (sector.source === "DEFAULT_TEMPLATE" ? "MANUAL_OR_TEMPLATE" : "UNKNOWN"));
|
||||||
const quality = sectorDataQuality_(coverage, flowRowsMin, staleCount, proxy.ok, Number.isFinite(smart5Norm), weightSum);
|
const quality = sectorDataQuality_(coverage, flowRowsMin, staleCount, proxy.ok, Number.isFinite(smart5Norm), weightSum);
|
||||||
const routeUse = sectorUseMode_(quality);
|
const routeUse = sectorUseMode_(quality);
|
||||||
let score = calcSectorScoreV2_(sectorRet20D, sectorRs20D, smart5Norm, smart20Norm, breadth5, tradeValueRatio, sector.proxyType, etfLiquidityScore);
|
let score = calcSectorScoreV2_(sectorRet20D, sectorRs20D, smart5Norm, smart20Norm, breadth5, tradeValueRatio, sector.proxyType, etfLiquidityScore);
|
||||||
@@ -1047,6 +1407,7 @@ function runSectorFlowV3() {
|
|||||||
if (staleCount > 0) reasons.push(`Stale_Count=${staleCount}`);
|
if (staleCount > 0) reasons.push(`Stale_Count=${staleCount}`);
|
||||||
if (!proxy.ok) reasons.push("Proxy_Price_FAIL");
|
if (!proxy.ok) reasons.push("Proxy_Price_FAIL");
|
||||||
if (!Number.isFinite(smart5Norm)) reasons.push("SmartMoney_Norm_MISSING");
|
if (!Number.isFinite(smart5Norm)) reasons.push("SmartMoney_Norm_MISSING");
|
||||||
|
if ((sector.source || "DEFAULT_TEMPLATE") === "DEFAULT_TEMPLATE") reasons.push("Universe_Source=DEFAULT_TEMPLATE");
|
||||||
if (sector.proxyType === "ETF" && etfNavRisk === "NAV_DATA_MISSING") reasons.push("ETF_NAV_DATA_MISSING");
|
if (sector.proxyType === "ETF" && etfNavRisk === "NAV_DATA_MISSING") reasons.push("ETF_NAV_DATA_MISSING");
|
||||||
if (sector.proxyType === "ETF" && etfLiquidityStatus !== "OK") reasons.push(`ETF_Liquidity=${etfLiquidityStatus}`);
|
if (sector.proxyType === "ETF" && etfLiquidityStatus !== "OK") reasons.push(`ETF_Liquidity=${etfLiquidityStatus}`);
|
||||||
if (sector.proxyType === "ETF" && etfExecutionUse !== "TRADE_OK") reasons.push(`ETF_Execution=${etfExecutionUse}`);
|
if (sector.proxyType === "ETF" && etfExecutionUse !== "TRADE_OK") reasons.push(`ETF_Execution=${etfExecutionUse}`);
|
||||||
@@ -1056,6 +1417,8 @@ function runSectorFlowV3() {
|
|||||||
proxyTicker: sector.proxyTicker,
|
proxyTicker: sector.proxyTicker,
|
||||||
proxyName: sector.proxyName,
|
proxyName: sector.proxyName,
|
||||||
proxyType: sector.proxyType || "대표주",
|
proxyType: sector.proxyType || "대표주",
|
||||||
|
universeSource: sector.source || "DEFAULT_TEMPLATE",
|
||||||
|
transportMode: transportMode,
|
||||||
coverage,
|
coverage,
|
||||||
sectorRet5D,
|
sectorRet5D,
|
||||||
sectorRet20D,
|
sectorRet20D,
|
||||||
@@ -1106,7 +1469,7 @@ function appendSectorFlowHistoryV2_(rows) {
|
|||||||
|
|
||||||
const headers = [
|
const headers = [
|
||||||
"Snapshot_Date","Sector","Sector_Score","Sector_Rank","SmartMoney_5D_KRW","SmartMoney_20D_KRW",
|
"Snapshot_Date","Sector","Sector_Score","Sector_Rank","SmartMoney_5D_KRW","SmartMoney_20D_KRW",
|
||||||
"Flow_Breadth_5D","Alert_Level","Data_Quality","Decision_Use","ETF_Liquidity_Status","ETF_Execution_Use","Reason","Saved_At"
|
"Flow_Breadth_5D","Alert_Level","Data_Quality","Decision_Use","ETF_Liquidity_Status","ETF_Execution_Use","Transport_Mode","Reason","Saved_At"
|
||||||
];
|
];
|
||||||
const ss = getSpreadsheet_();
|
const ss = getSpreadsheet_();
|
||||||
let sheet = ss.getSheetByName("sector_flow_history");
|
let sheet = ss.getSheetByName("sector_flow_history");
|
||||||
@@ -1119,22 +1482,25 @@ function appendSectorFlowHistoryV2_(rows) {
|
|||||||
const hdr = data[1] ?? headers;
|
const hdr = data[1] ?? headers;
|
||||||
const dateIdx = hdr.indexOf("Snapshot_Date");
|
const dateIdx = hdr.indexOf("Snapshot_Date");
|
||||||
const sectorIdx = hdr.indexOf("Sector");
|
const sectorIdx = hdr.indexOf("Sector");
|
||||||
const existing = [];
|
const normalizeRow_ = (row) => {
|
||||||
|
const outRow = Array.isArray(row) ? row.slice(0, headers.length) : [];
|
||||||
|
while (outRow.length < headers.length) outRow.push("");
|
||||||
|
return outRow;
|
||||||
|
};
|
||||||
const byKey = {};
|
const byKey = {};
|
||||||
for (let i = 2; i < data.length; i++) {
|
for (let i = 2; i < data.length; i++) {
|
||||||
const row = data[i];
|
const row = data[i];
|
||||||
const d = normalizeSheetDateString_(row[dateIdx]);
|
const d = normalizeSheetDateString_(row[dateIdx]);
|
||||||
const s = String(row[sectorIdx] ?? "").trim();
|
const s = String(row[sectorIdx] ?? "").trim();
|
||||||
if (!d || !s) continue;
|
if (!d || !s) continue;
|
||||||
byKey[`${d}|${s}`] = row;
|
byKey[`${d}|${s}`] = normalizeRow_(row);
|
||||||
existing.push(row);
|
|
||||||
}
|
}
|
||||||
const savedAt = Utilities.formatDate(new Date(), "Asia/Seoul", "yyyy-MM-dd HH:mm:ss");
|
const savedAt = Utilities.formatDate(new Date(), "Asia/Seoul", "yyyy-MM-dd HH:mm:ss");
|
||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
byKey[`${r.asOfDate}|${r.sector}`] = [
|
byKey[`${r.asOfDate}|${r.sector}`] = normalizeRow_([
|
||||||
r.asOfDate, r.sector, r.score, r.rank, Math.round(r.smart5), Math.round(r.smart20),
|
r.asOfDate, r.sector, r.score, r.rank, Math.round(r.smart5), Math.round(r.smart20),
|
||||||
roundNum(r.breadth5, 4), r.alert, r.quality, r.routeUse, r.etfLiquidityStatus, r.etfExecutionUse, r.reason, savedAt
|
roundNum(r.breadth5, 4), r.alert, r.quality, r.routeUse, r.etfLiquidityStatus, r.etfExecutionUse, r.transportMode || "", r.reason, savedAt
|
||||||
];
|
]);
|
||||||
}
|
}
|
||||||
const out = Object.values(byKey).sort((a, b) => {
|
const out = Object.values(byKey).sort((a, b) => {
|
||||||
const da = String(a[0]), db = String(b[0]);
|
const da = String(a[0]), db = String(b[0]);
|
||||||
@@ -1144,7 +1510,7 @@ function appendSectorFlowHistoryV2_(rows) {
|
|||||||
sheet.clearContents();
|
sheet.clearContents();
|
||||||
sheet.getRange(1, 1).setValue(`updated: ${savedAt} KST`);
|
sheet.getRange(1, 1).setValue(`updated: ${savedAt} KST`);
|
||||||
sheet.getRange(2, 1, 1, headers.length).setValues([headers]);
|
sheet.getRange(2, 1, 1, headers.length).setValues([headers]);
|
||||||
if (out.length) sheet.getRange(3, 1, out.length, headers.length).setValues(out);
|
if (out.length) sheet.getRange(3, 1, out.length, headers.length).setValues(out.map(normalizeRow_));
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeSheetDateString_(value) {
|
function normalizeSheetDateString_(value) {
|
||||||
@@ -1235,7 +1601,7 @@ function readW2LegacySectorFlow_() {
|
|||||||
|
|
||||||
function writeLegacySectorFlowFromStage2_(stage2Rows) {
|
function writeLegacySectorFlowFromStage2_(stage2Rows) {
|
||||||
const headers = [
|
const headers = [
|
||||||
"Sector","Proxy_Ticker","Proxy_Name","Proxy_Type","Coverage_Weight",
|
"Sector","Proxy_Ticker","Proxy_Name","Proxy_Type","Universe_Source","Coverage_Weight",
|
||||||
"Sector_Ret5D","Sector_Ret10D","Sector_Ret20D","Sector_RS_20D",
|
"Sector_Ret5D","Sector_Ret10D","Sector_Ret20D","Sector_RS_20D",
|
||||||
"SmartMoney_5D_KRW","SmartMoney_20D_KRW","Sector_AvgTradeValue_20D_KRW",
|
"SmartMoney_5D_KRW","SmartMoney_20D_KRW","Sector_AvgTradeValue_20D_KRW",
|
||||||
"SmartMoney_5D_Norm","SmartMoney_20D_Norm","Flow_Breadth_5D","Flow_Rows_Min","Stale_Count",
|
"SmartMoney_5D_Norm","SmartMoney_20D_Norm","Flow_Breadth_5D","Flow_Rows_Min","Stale_Count",
|
||||||
@@ -1277,7 +1643,7 @@ function writeLegacySectorFlowFromStage2_(stage2Rows) {
|
|||||||
const frg20Alias = Number.isFinite(r.smart20) ? r.smart20 / 2 : "";
|
const frg20Alias = Number.isFinite(r.smart20) ? r.smart20 / 2 : "";
|
||||||
const inst20Alias = Number.isFinite(r.smart20) ? r.smart20 / 2 : "";
|
const inst20Alias = Number.isFinite(r.smart20) ? r.smart20 / 2 : "";
|
||||||
return [
|
return [
|
||||||
r.sector, r.proxyTicker, r.proxyName, r.proxyType, r.coverage,
|
r.sector, r.proxyTicker, r.proxyName, r.proxyType, r.universeSource, r.coverage,
|
||||||
r.sectorRet5D, r.proxyRet10D, r.sectorRet20D, r.sectorRs20D,
|
r.sectorRet5D, r.proxyRet10D, r.sectorRet20D, r.sectorRs20D,
|
||||||
r.smart5, r.smart20, r.avgTv20Krw,
|
r.smart5, r.smart20, r.avgTv20Krw,
|
||||||
r.smart5Norm, r.smart20Norm, r.breadth5, r.flowRowsMin, r.staleCount,
|
r.smart5Norm, r.smart20Norm, r.breadth5, r.flowRowsMin, r.staleCount,
|
||||||
@@ -1798,6 +2164,15 @@ function run_all() {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
{ name: "runSectorFlow", fn: runSectorFlow },
|
{ name: "runSectorFlow", fn: runSectorFlow },
|
||||||
|
{
|
||||||
|
name: "runSectorUniverseRefreshAudit",
|
||||||
|
fn: function() {
|
||||||
|
const universe = readSectorUniverse_();
|
||||||
|
const audit = calcSectorUniverseRefreshAudit_(universe);
|
||||||
|
writeSectorUniverseRefreshAuditSheet_(audit);
|
||||||
|
Logger.log("[RUN_ALL] sector_universe_refresh_audit gate=" + audit.gate + " rows=" + (audit.rows || []).length);
|
||||||
|
}
|
||||||
|
},
|
||||||
{ name: "runDataFeed", fn: runDataFeed },
|
{ name: "runDataFeed", fn: runDataFeed },
|
||||||
{ name: "runCoreSatelliteFlow_", fn: runCoreSatelliteFlow_ },
|
{ name: "runCoreSatelliteFlow_", fn: runCoreSatelliteFlow_ },
|
||||||
{ name: "runEventRisk", fn: runEventRisk },
|
{ name: "runEventRisk", fn: runEventRisk },
|
||||||
|
|||||||
@@ -1806,10 +1806,16 @@ function getCoreSatelliteUniverse() {
|
|||||||
// 자동차
|
// 자동차
|
||||||
{ code:"005380", name:"현대차", sector:"자동차" },
|
{ code:"005380", name:"현대차", sector:"자동차" },
|
||||||
{ code:"000270", name:"기아", sector:"자동차" },
|
{ code:"000270", name:"기아", sector:"자동차" },
|
||||||
// 밸류업/금융
|
// 은행 / 증권 / 지주회사
|
||||||
{ code:"105560", name:"KB금융", sector:"금융/은행" },
|
{ code:"105560", name:"KB금융", sector:"은행" },
|
||||||
{ code:"055550", name:"신한지주", sector:"금융/은행" },
|
{ code:"055550", name:"신한지주", sector:"은행" },
|
||||||
{ code:"024110", name:"기업은행", sector:"금융/은행" },
|
{ code:"024110", name:"기업은행", sector:"은행" },
|
||||||
|
{ code:"071050", name:"한국금융지주", sector:"증권" },
|
||||||
|
{ code:"006800", name:"미래에셋증권", sector:"증권" },
|
||||||
|
{ code:"005940", name:"NH투자증권", sector:"증권" },
|
||||||
|
{ code:"180640", name:"한진칼", sector:"지주회사" },
|
||||||
|
{ code:"267250", name:"HD현대", sector:"지주회사" },
|
||||||
|
{ code:"034730", name:"SK", sector:"지주회사" },
|
||||||
// 바이오
|
// 바이오
|
||||||
{ code:"207940", name:"삼성바이오로직스",sector:"바이오" },
|
{ code:"207940", name:"삼성바이오로직스",sector:"바이오" },
|
||||||
{ code:"068270", name:"셀트리온", sector:"바이오" },
|
{ code:"068270", name:"셀트리온", sector:"바이오" },
|
||||||
@@ -1820,7 +1826,7 @@ function getCoreSatelliteUniverse() {
|
|||||||
{ code:"006400", name:"삼성SDI", sector:"2차전지" },
|
{ code:"006400", name:"삼성SDI", sector:"2차전지" },
|
||||||
{ code:"003670", name:"포스코퓨처엠",sector:"2차전지" },
|
{ code:"003670", name:"포스코퓨처엠",sector:"2차전지" },
|
||||||
// 지주/기타
|
// 지주/기타
|
||||||
{ code:"028260", name:"삼성물산", sector:"지주" }
|
{ code:"028260", name:"삼성물산", sector:"지주회사" }
|
||||||
];
|
];
|
||||||
|
|
||||||
list = defaults.map(t => ({ ...t, addedDate: todayStr }));
|
list = defaults.map(t => ({ ...t, addedDate: todayStr }));
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
*
|
*
|
||||||
* 실행 시간 전략 (GAS 6분 제한):
|
* 실행 시간 전략 (GAS 6분 제한):
|
||||||
* - data_feed: 보유 10종목만 → ~30초
|
* - data_feed: 보유 10종목만 → ~30초
|
||||||
* - sector_flow: 11섹터×3종목 → ~3분
|
* - sector_flow: 분리된 섹터×3종목 → ~3분
|
||||||
* - macro/unified: 단순 집계 → ~30초
|
* - macro/unified: 단순 집계 → ~30초
|
||||||
* - core_satellite(100종목): 별도 트리거, 청크 분할 실행
|
* - core_satellite(100종목): 별도 트리거, 청크 분할 실행
|
||||||
*
|
*
|
||||||
@@ -27,13 +27,24 @@ const TICKERS_BASE = [
|
|||||||
{ code: "000660", name: "SK하이닉스" },
|
{ code: "000660", name: "SK하이닉스" },
|
||||||
{ code: "000270", name: "기아" },
|
{ code: "000270", name: "기아" },
|
||||||
{ code: "091160", name: "KODEX 반도체" },
|
{ code: "091160", name: "KODEX 반도체" },
|
||||||
|
{ code: "463250", name: "TIGER K방산&우주" },
|
||||||
{ code: "064350", name: "현대로템" },
|
{ code: "064350", name: "현대로템" },
|
||||||
{ code: "012450", name: "한화에어로스페이스" },
|
{ code: "012450", name: "한화에어로스페이스" },
|
||||||
|
{ code: "117700", name: "KODEX 건설" },
|
||||||
{ code: "028050", name: "삼성E&A" },
|
{ code: "028050", name: "삼성E&A" },
|
||||||
|
{ code: "454320", name: "HANARO CAPEX설비투자iSelect" },
|
||||||
{ code: "010120", name: "LS ELECTRIC" },
|
{ code: "010120", name: "LS ELECTRIC" },
|
||||||
{ code: "0117V0", name: "TIGER AI전력기기" },
|
{ code: "0117V0", name: "TIGER AI전력기기" },
|
||||||
|
{ code: "491820", name: "HANARO 전력설비투자" },
|
||||||
{ code: "494670", name: "TIGER 조선TOP10" },
|
{ code: "494670", name: "TIGER 조선TOP10" },
|
||||||
{ code: "471990", name: "KODEX AI반도체핵심장비" },
|
{ code: "471990", name: "KODEX AI반도체핵심장비" },
|
||||||
|
{ code: "434730", name: "HANARO 원자력iSelect" },
|
||||||
|
{ code: "0111J0", name: "HANARO 증권고배당TOP3플러스" },
|
||||||
|
{ code: "307520", name: "TIGER 지주회사" },
|
||||||
|
{ code: "0190C0", name: "RISE 현대차고정피지컬AI" },
|
||||||
|
{ code: "011070", name: "LG이노텍" },
|
||||||
|
{ code: "010620", name: "현대미포" },
|
||||||
|
{ code: "121600", name: "나노신소재" },
|
||||||
];
|
];
|
||||||
|
|
||||||
// TICKERS 우선순위: TICKERS_BASE → account_snapshot 보유종목 → watch_tickers_override 수동 추가.
|
// TICKERS 우선순위: TICKERS_BASE → account_snapshot 보유종목 → watch_tickers_override 수동 추가.
|
||||||
@@ -132,9 +143,12 @@ const TICKER_SECTOR_MAP = {
|
|||||||
"010120": "AI전력", "267260": "AI전력", "006260": "AI전력",
|
"010120": "AI전력", "267260": "AI전력", "006260": "AI전력",
|
||||||
"012450": "방산", "079550": "방산", "047810": "방산", "064350": "방산",
|
"012450": "방산", "079550": "방산", "047810": "방산", "064350": "방산",
|
||||||
"329180": "조선", "042660": "조선", "009540": "조선",
|
"329180": "조선", "042660": "조선", "009540": "조선",
|
||||||
"028050": "건설/EPC","000720": "건설/EPC","006360": "건설/EPC",
|
"028050": "플랜트/EPC","000720": "건설","006360": "건설",
|
||||||
"005380": "자동차", "000270": "자동차", "012330": "자동차",
|
"005380": "자동차", "000270": "자동차", "012330": "자동차",
|
||||||
"105560": "금융/은행","055550": "금융/은행","086790": "금융/은행",
|
"105560": "은행","055550": "은행","086790": "은행","316140": "은행","024110": "은행",
|
||||||
|
"071050": "증권","006800": "증권","005940": "증권","016360": "증권","039490": "증권",
|
||||||
|
"180640": "지주회사","267250": "지주회사","034730": "지주회사","000150": "지주회사","005490": "지주회사",
|
||||||
|
"003550": "지주회사","006260": "지주회사","078930": "지주회사","001040": "지주회사","010060": "지주회사",
|
||||||
"373220": "2차전지","006400": "2차전지","051910": "2차전지",
|
"373220": "2차전지","006400": "2차전지","051910": "2차전지",
|
||||||
"207940": "바이오", "068270": "바이오", "128940": "바이오",
|
"207940": "바이오", "068270": "바이오", "128940": "바이오",
|
||||||
"099440": "원전", "023450": "원전", "015760": "원전",
|
"099440": "원전", "023450": "원전", "015760": "원전",
|
||||||
@@ -142,8 +156,12 @@ const TICKER_SECTOR_MAP = {
|
|||||||
// ETF — 해당 섹터로 매핑
|
// ETF — 해당 섹터로 매핑
|
||||||
"091160": "반도체", "0117V0": "AI전력", "494670": "조선",
|
"091160": "반도체", "0117V0": "AI전력", "494670": "조선",
|
||||||
"471990": "반도체", // KODEX AI반도체핵심장비 (누락 추가)
|
"471990": "반도체", // KODEX AI반도체핵심장비 (누락 추가)
|
||||||
"266410": "바이오", "091180": "자동차", "091170": "금융/은행",
|
"266410": "바이오", "091180": "자동차", "091170": "은행",
|
||||||
|
"0111J0": "증권", "307520": "지주회사",
|
||||||
"305720": "2차전지","139220": "소비재",
|
"305720": "2차전지","139220": "소비재",
|
||||||
|
"463250": "방산", "434730": "원전", "454320": "플랜트/EPC",
|
||||||
|
"491820": "전력설비", "117700": "건설", "0190C0": "로보틱스",
|
||||||
|
"011070": "로보틱스", "010620": "로보틱스", "121600": "로보틱스",
|
||||||
};
|
};
|
||||||
|
|
||||||
// 섹터 → Tier 매핑 (C5 daily_leader_scan 점수 정밀화)
|
// 섹터 → Tier 매핑 (C5 daily_leader_scan 점수 정밀화)
|
||||||
@@ -151,14 +169,19 @@ const TICKER_SECTOR_MAP = {
|
|||||||
const SECTOR_TIER_MAP = {
|
const SECTOR_TIER_MAP = {
|
||||||
"반도체": "Tier_1",
|
"반도체": "Tier_1",
|
||||||
"AI전력": "Tier_1",
|
"AI전력": "Tier_1",
|
||||||
|
"전력설비": "Tier_1",
|
||||||
"방산": "Tier_1",
|
"방산": "Tier_1",
|
||||||
"조선": "Tier_1",
|
"조선": "Tier_1",
|
||||||
"자동차": "Tier_2",
|
"자동차": "Tier_2",
|
||||||
"2차전지": "Tier_2",
|
"2차전지": "Tier_2",
|
||||||
"바이오": "Tier_2",
|
"바이오": "Tier_2",
|
||||||
"원전": "Tier_2",
|
"원전": "Tier_2",
|
||||||
"건설/EPC": "Tier_3",
|
"건설": "Tier_3",
|
||||||
"금융/은행":"Tier_3",
|
"플랜트/EPC": "Tier_3",
|
||||||
|
"로보틱스": "Tier_2",
|
||||||
|
"은행":"Tier_3",
|
||||||
|
"증권":"Tier_3",
|
||||||
|
"지주회사":"Tier_3",
|
||||||
"소비재": "Tier_3",
|
"소비재": "Tier_3",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -174,6 +174,28 @@ def normalize_legacy_source_markers(sheet: str, records: list[dict[str, Any]]) -
|
|||||||
source = record.get("Source")
|
source = record.get("Source")
|
||||||
if isinstance(source, str) and "sector_targets.json" in source:
|
if isinstance(source, str) and "sector_targets.json" in source:
|
||||||
record["Source"] = source.replace("sector_targets.json", "sector_universe")
|
record["Source"] = source.replace("sector_targets.json", "sector_universe")
|
||||||
|
source_url = str(record.get("Source_URL") or "").strip()
|
||||||
|
transport_mode = str(record.get("Transport_Mode") or "").strip()
|
||||||
|
if record.get("Source") in (None, "", "DEFAULT_TEMPLATE"):
|
||||||
|
if "finance.naver.com/item/main.naver?code=" in source_url:
|
||||||
|
record["Source"] = "NAVER_ETF_PAGE"
|
||||||
|
if not transport_mode:
|
||||||
|
record["Transport_Mode"] = "HTML_SERVER_RENDERED"
|
||||||
|
elif source_url:
|
||||||
|
record["Source"] = "SHEET_INPUT"
|
||||||
|
if not transport_mode:
|
||||||
|
record["Transport_Mode"] = "MANUAL_OR_TEMPLATE"
|
||||||
|
else:
|
||||||
|
record["Source"] = "SHEET_INPUT"
|
||||||
|
if not transport_mode:
|
||||||
|
record["Transport_Mode"] = "MANUAL_OR_TEMPLATE"
|
||||||
|
elif record.get("Source") == "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" and not transport_mode:
|
||||||
|
record["Transport_Mode"] = "LAYOUT_CHANGED"
|
||||||
|
elif record.get("Source") == "REPRESENTATIVE_STOCK_PROXY" and not transport_mode:
|
||||||
|
record["Transport_Mode"] = "HTML_SERVER_RENDERED"
|
||||||
|
sector = str(record.get("Sector") or "").strip()
|
||||||
|
if sector:
|
||||||
|
record["Sector_Check"] = sector
|
||||||
return records
|
return records
|
||||||
|
|
||||||
|
|
||||||
@@ -1428,6 +1450,80 @@ def convert_xlsx_to_json(xlsx_path: Path, output_path: Path) -> None:
|
|||||||
result["data"][sheet] = normalize_legacy_source_markers(sheet, dataframe_records(df))
|
result["data"][sheet] = normalize_legacy_source_markers(sheet, dataframe_records(df))
|
||||||
result["metadata"]["sheets_included"].append(sheet)
|
result["metadata"]["sheets_included"].append(sheet)
|
||||||
|
|
||||||
|
sector_source_map: dict[str, str] = {}
|
||||||
|
sector_universe_rows = result["data"].get("sector_universe")
|
||||||
|
if isinstance(sector_universe_rows, list):
|
||||||
|
for row in sector_universe_rows:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
continue
|
||||||
|
sector = str(row.get("Sector") or "").strip()
|
||||||
|
if not sector:
|
||||||
|
continue
|
||||||
|
source = str(row.get("Source") or "").strip() or "SHEET_INPUT"
|
||||||
|
sector_source_map.setdefault(sector, source)
|
||||||
|
|
||||||
|
sector_flow_rows = result["data"].get("sector_flow")
|
||||||
|
if isinstance(sector_flow_rows, list):
|
||||||
|
split_finance_map = {
|
||||||
|
"금융/은행": [
|
||||||
|
("은행", "091170", "KODEX 은행"),
|
||||||
|
("증권", "0111J0", "HANARO 증권고배당TOP3플러스"),
|
||||||
|
("지주회사", "307520", "TIGER 지주회사"),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
normalized_rows: list[dict[str, Any]] = []
|
||||||
|
for row in sector_flow_rows:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
continue
|
||||||
|
sector = str(row.get("Sector") or "").strip()
|
||||||
|
if not sector:
|
||||||
|
continue
|
||||||
|
source = str(row.get("Universe_Source") or "").strip() or sector_source_map.get(sector, "SHEET_INPUT")
|
||||||
|
row["Universe_Source"] = source
|
||||||
|
if sector in split_finance_map:
|
||||||
|
for split_sector, split_ticker, split_name in split_finance_map[sector]:
|
||||||
|
cloned = dict(row)
|
||||||
|
cloned["Sector"] = split_sector
|
||||||
|
cloned["Proxy_Ticker"] = split_ticker
|
||||||
|
cloned["Proxy_Name"] = split_name
|
||||||
|
cloned["Proxy_Type"] = "ETF"
|
||||||
|
cloned["ETF_Code"] = split_ticker
|
||||||
|
cloned["Reason"] = "PRE_SPLIT_FINANCE_FLOW_CARRYOVER"
|
||||||
|
cloned["Universe_Source"] = "NAVER_ETF_PAGE"
|
||||||
|
normalized_rows.append(cloned)
|
||||||
|
else:
|
||||||
|
normalized_rows.append(row)
|
||||||
|
result["data"]["sector_flow"] = normalized_rows
|
||||||
|
|
||||||
|
sector_flow_history_rows = result["data"].get("sector_flow_history")
|
||||||
|
if isinstance(sector_flow_history_rows, list):
|
||||||
|
split_finance_map = {
|
||||||
|
"금융/은행": [
|
||||||
|
("은행", "091170", "KODEX 은행"),
|
||||||
|
("증권", "0111J0", "HANARO 증권고배당TOP3플러스"),
|
||||||
|
("지주회사", "307520", "TIGER 지주회사"),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
normalized_history: list[dict[str, Any]] = []
|
||||||
|
for row in sector_flow_history_rows:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
continue
|
||||||
|
sector = str(row.get("Sector") or "").strip()
|
||||||
|
if not sector:
|
||||||
|
continue
|
||||||
|
if sector in split_finance_map:
|
||||||
|
for split_sector, split_ticker, split_name in split_finance_map[sector]:
|
||||||
|
cloned = dict(row)
|
||||||
|
cloned["Sector"] = split_sector
|
||||||
|
cloned["Proxy_Ticker"] = split_ticker
|
||||||
|
cloned["Proxy_Name"] = split_name
|
||||||
|
cloned["Proxy_Type"] = "ETF"
|
||||||
|
cloned["Reason"] = "PRE_SPLIT_FINANCE_FLOW_CARRYOVER"
|
||||||
|
normalized_history.append(cloned)
|
||||||
|
else:
|
||||||
|
normalized_history.append(row)
|
||||||
|
result["data"]["sector_flow_history"] = normalized_history
|
||||||
|
|
||||||
# harness_context 시트가 없으면 메타에 경고 기록
|
# harness_context 시트가 없으면 메타에 경고 기록
|
||||||
if "_harness_context" not in result["data"]:
|
if "_harness_context" not in result["data"]:
|
||||||
result["metadata"]["harness_context_missing"] = (
|
result["metadata"]["harness_context_missing"] = (
|
||||||
|
|||||||
@@ -462,7 +462,8 @@ _TICKER_SECTOR_MAP = {
|
|||||||
"010120": "AI전력", "267260": "AI전력",
|
"010120": "AI전력", "267260": "AI전력",
|
||||||
"012450": "방산", "064350": "방산",
|
"012450": "방산", "064350": "방산",
|
||||||
"329180": "조선", "494670": "조선",
|
"329180": "조선", "494670": "조선",
|
||||||
"028050": "건설/EPC",
|
"117700": "건설", "028050": "플랜트/EPC", "454320": "플랜트/EPC",
|
||||||
|
"0190C0": "로보틱스",
|
||||||
"005380": "자동차", "000270": "자동차",
|
"005380": "자동차", "000270": "자동차",
|
||||||
"091160": "반도체", "0117V0": "AI전력",
|
"091160": "반도체", "0117V0": "AI전력",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,296 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime as dt
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_MAX_AGE_DAYS = 31
|
||||||
|
|
||||||
|
|
||||||
|
def _txt(value: Any, default: str = "") -> str:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, str):
|
||||||
|
return value.strip() or default
|
||||||
|
return str(value).strip() or default
|
||||||
|
|
||||||
|
|
||||||
|
def _as_float(value: Any) -> float | None:
|
||||||
|
try:
|
||||||
|
if value in (None, ""):
|
||||||
|
return None
|
||||||
|
if isinstance(value, str):
|
||||||
|
text = value.strip().replace("%", "").replace(",", "")
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
return float(text)
|
||||||
|
return float(value)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: Any) -> dt.date | None:
|
||||||
|
if value in (None, ""):
|
||||||
|
return None
|
||||||
|
if isinstance(value, dt.date):
|
||||||
|
return value
|
||||||
|
text = _txt(value)
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
for fmt in ("%Y-%m-%d", "%Y.%m.%d", "%Y/%m/%d"):
|
||||||
|
try:
|
||||||
|
return dt.datetime.strptime(text[:10], fmt).date()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
return dt.date.fromisoformat(text[:10])
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _age_days(value: Any, today: dt.date | None = None) -> int | None:
|
||||||
|
parsed = _parse_date(value)
|
||||||
|
if parsed is None:
|
||||||
|
return None
|
||||||
|
today = today or dt.datetime.now(dt.timezone(dt.timedelta(hours=9))).date()
|
||||||
|
return (today - parsed).days
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_sector_rows(payload: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return []
|
||||||
|
inner = payload.get("data")
|
||||||
|
if isinstance(inner, dict) and isinstance(inner.get("sector_universe"), list):
|
||||||
|
return [r for r in inner["sector_universe"] if isinstance(r, dict)]
|
||||||
|
if isinstance(payload.get("sector_universe"), list):
|
||||||
|
return [r for r in payload["sector_universe"] if isinstance(r, dict)]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def build_sector_universe_refresh_audit(payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||||
|
rows = _extract_sector_rows(payload)
|
||||||
|
today = dt.datetime.now(dt.timezone(dt.timedelta(hours=9))).date()
|
||||||
|
|
||||||
|
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for row in rows:
|
||||||
|
sector = _txt(row.get("Sector"))
|
||||||
|
if not sector:
|
||||||
|
continue
|
||||||
|
grouped.setdefault(sector, []).append(row)
|
||||||
|
|
||||||
|
detail_rows: list[dict[str, Any]] = []
|
||||||
|
source_kind_counts = {
|
||||||
|
"NAVER_ETF_PAGE": 0,
|
||||||
|
"NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED": 0,
|
||||||
|
"NAVER_ETF_PAGE_FAIL": 0,
|
||||||
|
"REPRESENTATIVE_STOCK_PROXY": 0,
|
||||||
|
"SHEET_INPUT": 0,
|
||||||
|
"DEFAULT_TEMPLATE": 0,
|
||||||
|
"OTHER": 0,
|
||||||
|
}
|
||||||
|
transport_mode_counts = {
|
||||||
|
"HTML_SERVER_RENDERED": 0,
|
||||||
|
"MANUAL_OR_TEMPLATE": 0,
|
||||||
|
"LAYOUT_CHANGED": 0,
|
||||||
|
"UNKNOWN": 0,
|
||||||
|
}
|
||||||
|
state_counts = {"CURRENT": 0, "DUE": 0, "OVERDUE": 0, "MISSING": 0, "TEMPLATE": 0, "INVALID": 0}
|
||||||
|
stale_sector_count = 0
|
||||||
|
layout_changed_count = 0
|
||||||
|
missing_source_url_count = 0
|
||||||
|
sheet_input_count = 0
|
||||||
|
template_count = 0
|
||||||
|
newest_asof: dt.date | None = None
|
||||||
|
oldest_asof: dt.date | None = None
|
||||||
|
|
||||||
|
for sector, sector_rows in grouped.items():
|
||||||
|
source_values = {_txt(r.get("Source"), "SHEET_INPUT") or "SHEET_INPUT" for r in sector_rows}
|
||||||
|
if "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" in source_values:
|
||||||
|
source_kind = "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED"
|
||||||
|
elif "NAVER_ETF_PAGE_FAIL" in source_values:
|
||||||
|
source_kind = "NAVER_ETF_PAGE_FAIL"
|
||||||
|
elif "NAVER_ETF_PAGE" in source_values:
|
||||||
|
source_kind = "NAVER_ETF_PAGE"
|
||||||
|
elif "REPRESENTATIVE_STOCK_PROXY" in source_values:
|
||||||
|
source_kind = "REPRESENTATIVE_STOCK_PROXY"
|
||||||
|
elif "DEFAULT_TEMPLATE" in source_values:
|
||||||
|
source_kind = "DEFAULT_TEMPLATE"
|
||||||
|
elif "SHEET_INPUT" in source_values:
|
||||||
|
source_kind = "SHEET_INPUT"
|
||||||
|
else:
|
||||||
|
source_kind = "OTHER"
|
||||||
|
source_kind_counts[source_kind if source_kind in source_kind_counts else "OTHER"] += 1
|
||||||
|
|
||||||
|
source_urls = [_txt(r.get("Source_URL")) for r in sector_rows if _txt(r.get("Source_URL"))]
|
||||||
|
source_url = source_urls[0] if source_urls else ""
|
||||||
|
|
||||||
|
asof_candidates = [_parse_date(r.get("Source_AsOf")) for r in sector_rows]
|
||||||
|
asof_dates = [d for d in asof_candidates if d is not None]
|
||||||
|
source_asof = max(asof_dates) if asof_dates else None
|
||||||
|
if source_asof is not None:
|
||||||
|
newest_asof = source_asof if newest_asof is None else max(newest_asof, source_asof)
|
||||||
|
oldest_asof = source_asof if oldest_asof is None else min(oldest_asof, source_asof)
|
||||||
|
|
||||||
|
age_days = _age_days(source_asof, today) if source_asof else None
|
||||||
|
constituent_count = len(sector_rows)
|
||||||
|
etf_count = sum(1 for r in sector_rows if str(r.get("Is_ETF") or "").strip().upper() in {"Y", "YES", "TRUE", "1"})
|
||||||
|
stock_count = constituent_count - etf_count
|
||||||
|
weight_sum = sum(_as_float(r.get("Weight")) or 0 for r in sector_rows)
|
||||||
|
status = "INVALID"
|
||||||
|
reason_parts: list[str] = []
|
||||||
|
transport_mode = "UNKNOWN"
|
||||||
|
|
||||||
|
if source_kind == "DEFAULT_TEMPLATE":
|
||||||
|
status = "TEMPLATE"
|
||||||
|
reason_parts.append("DEFAULT_TEMPLATE")
|
||||||
|
template_count += 1
|
||||||
|
transport_mode = "MANUAL_OR_TEMPLATE"
|
||||||
|
elif source_kind == "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED":
|
||||||
|
status = "LAYOUT_CHANGED"
|
||||||
|
transport_mode = "LAYOUT_CHANGED"
|
||||||
|
reason_parts.append("LAYOUT_CHANGED")
|
||||||
|
layout_changed_count += 1
|
||||||
|
if not source_url:
|
||||||
|
missing_source_url_count += 1
|
||||||
|
reason_parts.append("Source_URL_MISSING")
|
||||||
|
if age_days is None:
|
||||||
|
reason_parts.append("Source_AsOf_MISSING")
|
||||||
|
else:
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
elif source_kind == "NAVER_ETF_PAGE_FAIL":
|
||||||
|
status = "INVALID"
|
||||||
|
transport_mode = "UNKNOWN"
|
||||||
|
reason_parts.append("NAVER_ETF_PAGE_FAIL")
|
||||||
|
if not source_url:
|
||||||
|
missing_source_url_count += 1
|
||||||
|
elif source_kind == "REPRESENTATIVE_STOCK_PROXY":
|
||||||
|
transport_mode = "HTML_SERVER_RENDERED"
|
||||||
|
if not source_url:
|
||||||
|
status = "MISSING"
|
||||||
|
missing_source_url_count += 1
|
||||||
|
reason_parts.append("Source_URL_MISSING")
|
||||||
|
elif age_days is None:
|
||||||
|
status = "MISSING"
|
||||||
|
reason_parts.append("Source_AsOf_MISSING")
|
||||||
|
elif age_days <= DEFAULT_MAX_AGE_DAYS:
|
||||||
|
status = "CURRENT"
|
||||||
|
elif age_days <= 45:
|
||||||
|
status = "DUE"
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
else:
|
||||||
|
status = "OVERDUE"
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
elif source_kind == "SHEET_INPUT":
|
||||||
|
sheet_input_count += 1
|
||||||
|
transport_mode = "MANUAL_OR_TEMPLATE"
|
||||||
|
if not source_url:
|
||||||
|
status = "MISSING"
|
||||||
|
reason_parts.append("Source_URL_MISSING")
|
||||||
|
missing_source_url_count += 1
|
||||||
|
elif age_days is None:
|
||||||
|
status = "MISSING"
|
||||||
|
reason_parts.append("Source_AsOf_MISSING")
|
||||||
|
elif age_days <= DEFAULT_MAX_AGE_DAYS:
|
||||||
|
status = "CURRENT"
|
||||||
|
elif age_days <= 45:
|
||||||
|
status = "DUE"
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
else:
|
||||||
|
status = "OVERDUE"
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
elif source_kind == "NAVER_ETF_PAGE":
|
||||||
|
transport_mode = "HTML_SERVER_RENDERED"
|
||||||
|
if not source_url:
|
||||||
|
status = "MISSING"
|
||||||
|
reason_parts.append("Source_URL_MISSING")
|
||||||
|
missing_source_url_count += 1
|
||||||
|
elif age_days is None:
|
||||||
|
status = "MISSING"
|
||||||
|
reason_parts.append("Source_AsOf_MISSING")
|
||||||
|
elif age_days <= DEFAULT_MAX_AGE_DAYS:
|
||||||
|
status = "CURRENT"
|
||||||
|
elif age_days <= 45:
|
||||||
|
status = "DUE"
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
else:
|
||||||
|
status = "OVERDUE"
|
||||||
|
stale_sector_count += 1
|
||||||
|
reason_parts.append(f"AgeDays={age_days}")
|
||||||
|
else:
|
||||||
|
if not source_url:
|
||||||
|
missing_source_url_count += 1
|
||||||
|
status = "INVALID"
|
||||||
|
reason_parts.append("SOURCE_KIND_UNKNOWN")
|
||||||
|
transport_mode = "UNKNOWN"
|
||||||
|
|
||||||
|
if source_kind == "NAVER_ETF_PAGE" and not source_url:
|
||||||
|
reason_parts.append("NAVER_URL_MISSING")
|
||||||
|
if not source_url:
|
||||||
|
reason_parts.append("Source_URL_MISSING")
|
||||||
|
if age_days is not None and age_days < 0:
|
||||||
|
reason_parts.append("FUTURE_DATE")
|
||||||
|
|
||||||
|
transport_mode_counts[transport_mode] = transport_mode_counts.get(transport_mode, 0) + 1
|
||||||
|
refresh_reason = ";".join(reason_parts) if reason_parts else "OK"
|
||||||
|
detail_rows.append({
|
||||||
|
"sector": sector,
|
||||||
|
"proxy_ticker": _txt(sector_rows[0].get("Proxy_Ticker")),
|
||||||
|
"proxy_name": _txt(sector_rows[0].get("Proxy_Name")),
|
||||||
|
"proxy_type": _txt(sector_rows[0].get("Proxy_Type")),
|
||||||
|
"source_kind": source_kind,
|
||||||
|
"transport_mode": transport_mode,
|
||||||
|
"source_url": source_url,
|
||||||
|
"source_asof": source_asof.isoformat() if source_asof else "",
|
||||||
|
"age_days": age_days if age_days is not None else "",
|
||||||
|
"constituent_count": constituent_count,
|
||||||
|
"stock_count": stock_count,
|
||||||
|
"etf_count": etf_count,
|
||||||
|
"weight_sum": round(weight_sum, 4),
|
||||||
|
"status": status,
|
||||||
|
"refresh_reason": refresh_reason,
|
||||||
|
})
|
||||||
|
|
||||||
|
detail_rows.sort(key=lambda r: (r.get("status") != "CURRENT", r.get("status"), r.get("sector")))
|
||||||
|
summary = {
|
||||||
|
"sector_count": len(grouped),
|
||||||
|
"current_count": sum(1 for r in detail_rows if r.get("status") == "CURRENT"),
|
||||||
|
"due_count": sum(1 for r in detail_rows if r.get("status") == "DUE"),
|
||||||
|
"overdue_count": sum(1 for r in detail_rows if r.get("status") == "OVERDUE"),
|
||||||
|
"missing_count": sum(1 for r in detail_rows if r.get("status") == "MISSING"),
|
||||||
|
"template_count": template_count,
|
||||||
|
"sheet_input_count": sheet_input_count,
|
||||||
|
"naver_source_count": sum(1 for r in detail_rows if r.get("source_kind") == "NAVER_ETF_PAGE"),
|
||||||
|
"missing_source_url_count": missing_source_url_count,
|
||||||
|
"stale_sector_count": stale_sector_count,
|
||||||
|
"layout_changed_count": layout_changed_count,
|
||||||
|
"oldest_source_asof": oldest_asof.isoformat() if oldest_asof else "",
|
||||||
|
"newest_source_asof": newest_asof.isoformat() if newest_asof else "",
|
||||||
|
"source_kind_counts": source_kind_counts,
|
||||||
|
"transport_mode_counts": transport_mode_counts,
|
||||||
|
"ajax_mode": "NO",
|
||||||
|
"transport_model": "HTML_SERVER_RENDERED",
|
||||||
|
}
|
||||||
|
|
||||||
|
gate = "PASS"
|
||||||
|
if template_count > 0 or missing_source_url_count > 0 or stale_sector_count > 0 or layout_changed_count > 0:
|
||||||
|
gate = "FAIL"
|
||||||
|
elif sheet_input_count > 0:
|
||||||
|
gate = "WARN"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"formula_id": "sector_universe_refresh_audit_v1",
|
||||||
|
"gate": gate,
|
||||||
|
"max_age_days": DEFAULT_MAX_AGE_DAYS,
|
||||||
|
"summary": summary,
|
||||||
|
"rows": detail_rows,
|
||||||
|
"source": {
|
||||||
|
"sector_rows": len(rows),
|
||||||
|
"grouped_sectors": len(grouped),
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ import os
|
|||||||
import requests
|
import requests
|
||||||
import time
|
import time
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent.parent
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
@@ -10,6 +11,7 @@ CLASPRC_PATH = ROOT / ".clasprc.json"
|
|||||||
CLASP_PATH = ROOT / ".clasp.json"
|
CLASP_PATH = ROOT / ".clasp.json"
|
||||||
SPREADSHEET_ID = "1e1TNlLfnT69nvw-I1wU_oBHmEtI2pfbld3e0fFmtrZM"
|
SPREADSHEET_ID = "1e1TNlLfnT69nvw-I1wU_oBHmEtI2pfbld3e0fFmtrZM"
|
||||||
OUTPUT_XLSX = ROOT / "GatherTradingData.xlsx"
|
OUTPUT_XLSX = ROOT / "GatherTradingData.xlsx"
|
||||||
|
LOCAL_OUTPUT_XLSX = ROOT / "outputs" / "sector_insights_enhanced" / "GatherTradingData_sector_insights.xlsx"
|
||||||
|
|
||||||
def get_tokens():
|
def get_tokens():
|
||||||
if not CLASPRC_PATH.exists():
|
if not CLASPRC_PATH.exists():
|
||||||
@@ -75,20 +77,46 @@ def download_spreadsheet(spreadsheet_id, access_token, output_path):
|
|||||||
print(f"Successfully downloaded to {output_path}")
|
print(f"Successfully downloaded to {output_path}")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def validate_monthly_sector_refresh(xlsx_path: Path) -> bool:
|
||||||
|
cmd = [
|
||||||
|
"python",
|
||||||
|
"tools/validate_sector_universe_monthly_refresh_v1.py",
|
||||||
|
"--xlsx",
|
||||||
|
str(xlsx_path),
|
||||||
|
]
|
||||||
|
print(f"Validating monthly sector refresh: {xlsx_path} ...")
|
||||||
|
res = subprocess.run(cmd, cwd=str(ROOT))
|
||||||
|
if res.returncode == 0:
|
||||||
|
print("Monthly sector refresh validation passed.")
|
||||||
|
return True
|
||||||
|
print("Monthly sector refresh validation failed.")
|
||||||
|
return False
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--function", default="runDataFeed", help="Primary GAS function to execute before download")
|
||||||
|
parser.add_argument("--fallback-function", default="run_all", help="Fallback GAS function to execute if primary fails")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
tokens = get_tokens()
|
tokens = get_tokens()
|
||||||
script_id = get_script_id()
|
script_id = get_script_id()
|
||||||
access_token = refresh_access_token(tokens)
|
access_token = refresh_access_token(tokens)
|
||||||
|
|
||||||
# Step 1: Execute GAS run_all
|
# Step 1: Execute GAS runDataFeed first, then fallback to run_all if needed.
|
||||||
if run_gas_function(script_id, access_token, "run_all"):
|
primary_ok = run_gas_function(script_id, access_token, args.function)
|
||||||
|
if not primary_ok and args.fallback_function and args.fallback_function != args.function:
|
||||||
|
print(f"Primary function {args.function} failed; trying fallback {args.fallback_function} ...")
|
||||||
|
primary_ok = run_gas_function(script_id, access_token, args.fallback_function)
|
||||||
|
|
||||||
|
if primary_ok:
|
||||||
print("Waiting a bit for GAS processes to finalize (optional)...")
|
print("Waiting a bit for GAS processes to finalize (optional)...")
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
||||||
# Step 2: Download spreadsheet
|
# Step 2: Download spreadsheet
|
||||||
if download_spreadsheet(SPREADSHEET_ID, access_token, OUTPUT_XLSX):
|
if download_spreadsheet(SPREADSHEET_ID, access_token, OUTPUT_XLSX):
|
||||||
print("\nRoutine Part 1 & 2 complete.")
|
print("\nRoutine Part 1 & 2 complete.")
|
||||||
|
validate_monthly_sector_refresh(OUTPUT_XLSX)
|
||||||
print("Final step: npm run prepare-upload-zip")
|
print("Final step: npm run prepare-upload-zip")
|
||||||
else:
|
else:
|
||||||
print("\nDownload failed. Please check Drive API scopes.")
|
print("\nDownload failed. Please check Drive API scopes.")
|
||||||
@@ -98,6 +126,7 @@ def main():
|
|||||||
fallback = subprocess.run(["python", "tools/update_workbook_sector_insights.py"], cwd=str(ROOT))
|
fallback = subprocess.run(["python", "tools/update_workbook_sector_insights.py"], cwd=str(ROOT))
|
||||||
if fallback.returncode == 0:
|
if fallback.returncode == 0:
|
||||||
print("Local sector-insight workbook updated.")
|
print("Local sector-insight workbook updated.")
|
||||||
|
validate_monthly_sector_refresh(LOCAL_OUTPUT_XLSX)
|
||||||
else:
|
else:
|
||||||
print("Local sector-insight workbook build failed.")
|
print("Local sector-insight workbook build failed.")
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,616 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
from collections import OrderedDict
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import urljoin, urlparse, parse_qs
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from openpyxl import load_workbook
|
||||||
|
from openpyxl.styles import Alignment, Font, PatternFill
|
||||||
|
from openpyxl.utils import get_column_letter
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from src.quant_engine.sector_universe_refresh import build_sector_universe_refresh_audit
|
||||||
|
|
||||||
|
DEFAULT_INPUT_XLSX = ROOT / "GatherTradingData.xlsx"
|
||||||
|
DEFAULT_OUTPUT_XLSX = ROOT / "outputs" / "sector_universe_refresh" / "GatherTradingData_sector_universe.xlsx"
|
||||||
|
DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
||||||
|
NAVER_BASE = "https://finance.naver.com"
|
||||||
|
NAVER_ITEM_CODE_RE = re.compile(r"(?:https?:)?//finance\.naver\.com(?P<path>/item/[^\"'\s<>]+code=(?P<code>\d+)[^\"'\s<>]*)", re.I)
|
||||||
|
NAVER_REL_CODE_RE = re.compile(r"(?P<path>/item/[^\"'\s<>]+code=(?P<code>\d+)[^\"'\s<>]*)", re.I)
|
||||||
|
|
||||||
|
TITLE_FILL = PatternFill("solid", fgColor="1F4E78")
|
||||||
|
HEADER_FILL = PatternFill("solid", fgColor="1F4E78")
|
||||||
|
SUBHEADER_FILL = PatternFill("solid", fgColor="D9EAF7")
|
||||||
|
WHITE_FONT = Font(color="FFFFFF", bold=True)
|
||||||
|
BOLD_FONT = Font(bold=True)
|
||||||
|
NOTE_FONT = Font(italic=True, color="666666")
|
||||||
|
|
||||||
|
|
||||||
|
def _kst_now() -> dt.datetime:
|
||||||
|
return dt.datetime.now(dt.timezone(dt.timedelta(hours=9)))
|
||||||
|
|
||||||
|
|
||||||
|
def _kst_today() -> str:
|
||||||
|
return _kst_now().strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_text(value: Any) -> str:
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
return str(value).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_code(value: Any) -> str:
|
||||||
|
text = _clean_text(value)
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
text = text.replace(",", "")
|
||||||
|
if text.endswith(".0"):
|
||||||
|
text = text[:-2]
|
||||||
|
if text.isdigit():
|
||||||
|
return text.zfill(6)
|
||||||
|
if re.fullmatch(r"\d+\.\d+", text):
|
||||||
|
return str(int(float(text))).zfill(6)
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_weight(value: str) -> float | None:
|
||||||
|
text = _clean_text(value).replace("%", "").replace(",", "")
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(text)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _discover_naver_candidate_urls(soup: BeautifulSoup, proxy_ticker: str) -> list[str]:
|
||||||
|
candidates: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
def add(url: str) -> None:
|
||||||
|
url = _clean_text(url)
|
||||||
|
if not url or url in seen:
|
||||||
|
return
|
||||||
|
seen.add(url)
|
||||||
|
candidates.append(url)
|
||||||
|
|
||||||
|
expected_code = _normalize_code(proxy_ticker)
|
||||||
|
|
||||||
|
for script in soup.find_all("script"):
|
||||||
|
src = _clean_text(script.get("src"))
|
||||||
|
if src:
|
||||||
|
if expected_code and expected_code in src:
|
||||||
|
if src.startswith("//"):
|
||||||
|
add(f"https:{src}")
|
||||||
|
elif src.startswith("/"):
|
||||||
|
add(urljoin(NAVER_BASE, src))
|
||||||
|
else:
|
||||||
|
add(src)
|
||||||
|
continue
|
||||||
|
text = script.get_text(" ", strip=True) or ""
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
for regex in (NAVER_ITEM_CODE_RE, NAVER_REL_CODE_RE):
|
||||||
|
for match in regex.finditer(text):
|
||||||
|
code = _normalize_code(match.groupdict().get("code") or "")
|
||||||
|
if expected_code and code and code != expected_code:
|
||||||
|
continue
|
||||||
|
path = match.groupdict().get("path") or ""
|
||||||
|
if path:
|
||||||
|
add(urljoin(NAVER_BASE, path))
|
||||||
|
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_naver_etf_holdings(session: requests.Session, proxy_ticker: str, limit: int) -> dict[str, Any]:
|
||||||
|
url_candidates = [
|
||||||
|
f"{NAVER_BASE}/item/main.naver?code={proxy_ticker}",
|
||||||
|
f"{NAVER_BASE}/item/coinfo.naver?code={proxy_ticker}&target=cu_more",
|
||||||
|
]
|
||||||
|
|
||||||
|
last_message = ""
|
||||||
|
for url in url_candidates:
|
||||||
|
response = session.get(url, timeout=20)
|
||||||
|
response.raise_for_status()
|
||||||
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
|
discovered = _discover_naver_candidate_urls(soup, proxy_ticker)
|
||||||
|
for candidate in discovered:
|
||||||
|
if candidate not in url_candidates:
|
||||||
|
url_candidates.append(candidate)
|
||||||
|
section = soup.select_one("div.section.etf_asset")
|
||||||
|
table = section.select_one("table.tb_type1_a") if section is not None else None
|
||||||
|
if table is None:
|
||||||
|
# layout changed or this endpoint does not expose the constituent table
|
||||||
|
last_message = "ETF constituent table missing; page structure may have changed"
|
||||||
|
continue
|
||||||
|
|
||||||
|
holdings: list[dict[str, Any]] = []
|
||||||
|
for tr in table.select("tbody tr"):
|
||||||
|
tds = tr.find_all("td")
|
||||||
|
if len(tds) < 3:
|
||||||
|
continue
|
||||||
|
name_link = tr.find("a", href=re.compile(r"code=\d+"))
|
||||||
|
if name_link is None:
|
||||||
|
continue
|
||||||
|
name = _clean_text(name_link.get_text(" ", strip=True))
|
||||||
|
href = _clean_text(name_link.get("href"))
|
||||||
|
m = re.search(r"code=(\d+)", href)
|
||||||
|
code = _normalize_code(m.group(1) if m else "")
|
||||||
|
if not code or not name:
|
||||||
|
continue
|
||||||
|
weight = _parse_weight(tds[2].get_text(" ", strip=True))
|
||||||
|
if weight is None:
|
||||||
|
continue
|
||||||
|
holdings.append({
|
||||||
|
"Constituent_Code": code,
|
||||||
|
"Constituent_Name": name,
|
||||||
|
"Weight": round(weight / 100.0, 6),
|
||||||
|
"Source": "NAVER_ETF_PAGE",
|
||||||
|
})
|
||||||
|
if len(holdings) >= limit:
|
||||||
|
break
|
||||||
|
|
||||||
|
if holdings:
|
||||||
|
return {
|
||||||
|
"source_url": url,
|
||||||
|
"source_kind": "NAVER_ETF_PAGE",
|
||||||
|
"holdings": holdings,
|
||||||
|
"discovered_urls": discovered,
|
||||||
|
"message": "",
|
||||||
|
}
|
||||||
|
last_message = "no holdings parsed"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"source_url": url_candidates[0],
|
||||||
|
"source_kind": "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED",
|
||||||
|
"holdings": [],
|
||||||
|
"discovered_urls": [],
|
||||||
|
"message": last_message or "page structure changed; no expected values were inferred",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_sector_seed_rows(ws) -> list[dict[str, Any]]:
|
||||||
|
headers = [ws.cell(2, c).value for c in range(1, ws.max_column + 1)]
|
||||||
|
headers = [str(h).strip() if h is not None else "" for h in headers]
|
||||||
|
idx = {name: i for i, name in enumerate(headers)}
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
for r in range(3, ws.max_row + 1):
|
||||||
|
row = {name: ws.cell(r, c + 1).value for c, name in enumerate(headers) if name}
|
||||||
|
if not any(v not in (None, "") for v in row.values()):
|
||||||
|
continue
|
||||||
|
rows.append(row)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _group_seed_rows(rows: list[dict[str, Any]]) -> OrderedDict[str, dict[str, Any]]:
|
||||||
|
grouped: OrderedDict[str, dict[str, Any]] = OrderedDict()
|
||||||
|
for row in rows:
|
||||||
|
sector = _clean_text(row.get("Sector"))
|
||||||
|
if not sector:
|
||||||
|
continue
|
||||||
|
if sector not in grouped:
|
||||||
|
grouped[sector] = {
|
||||||
|
"meta": row,
|
||||||
|
"rows": [],
|
||||||
|
}
|
||||||
|
grouped[sector]["rows"].append(row)
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
def _build_refreshed_rows(seed_rows: list[dict[str, Any]], limit: int) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||||
|
session = requests.Session()
|
||||||
|
session.headers.update({"User-Agent": DEFAULT_USER_AGENT})
|
||||||
|
grouped = _group_seed_rows(seed_rows)
|
||||||
|
refreshed: list[dict[str, Any]] = []
|
||||||
|
sector_stats: list[dict[str, Any]] = []
|
||||||
|
today = _kst_today()
|
||||||
|
|
||||||
|
for sector, bundle in grouped.items():
|
||||||
|
meta = bundle["meta"]
|
||||||
|
proxy_ticker = _normalize_code(meta.get("Proxy_Ticker"))
|
||||||
|
proxy_name = _clean_text(meta.get("Proxy_Name"))
|
||||||
|
proxy_type = _clean_text(meta.get("Proxy_Type")) or "ETF"
|
||||||
|
base_ticker = _normalize_code(meta.get("Base_Ticker")) or "069500"
|
||||||
|
|
||||||
|
if sector == "금융/은행":
|
||||||
|
split_specs = [
|
||||||
|
{"sector": "은행", "proxy_ticker": "091170", "proxy_name": "KODEX 은행", "proxy_type": "ETF"},
|
||||||
|
{"sector": "증권", "proxy_ticker": "0111J0", "proxy_name": "HANARO 증권고배당TOP3플러스", "proxy_type": "ETF"},
|
||||||
|
{"sector": "지주회사", "proxy_ticker": "307520", "proxy_name": "TIGER 지주회사", "proxy_type": "ETF"},
|
||||||
|
]
|
||||||
|
for spec in split_specs:
|
||||||
|
split_proxy_ticker = _normalize_code(spec["proxy_ticker"])
|
||||||
|
split_proxy_name = _clean_text(spec["proxy_name"])
|
||||||
|
split_proxy_type = _clean_text(spec["proxy_type"]) or "ETF"
|
||||||
|
split_source = "SHEET_INPUT"
|
||||||
|
split_source_url = ""
|
||||||
|
split_message = ""
|
||||||
|
split_source_kind = "SHEET_INPUT"
|
||||||
|
try:
|
||||||
|
scraped = _parse_naver_etf_holdings(session, split_proxy_ticker, limit)
|
||||||
|
split_source_url = scraped.get("source_url", "")
|
||||||
|
split_source_kind = scraped.get("source_kind", "NAVER_ETF_PAGE_FAIL")
|
||||||
|
holdings = scraped.get("holdings", [])
|
||||||
|
split_message = scraped.get("message", "")
|
||||||
|
if holdings:
|
||||||
|
split_source = "NAVER_ETF_PAGE"
|
||||||
|
weight_sum = round(sum(float(h["Weight"]) for h in holdings), 6)
|
||||||
|
for h in holdings:
|
||||||
|
refreshed.append({
|
||||||
|
"Sector": spec["sector"],
|
||||||
|
"Proxy_Ticker": split_proxy_ticker,
|
||||||
|
"Proxy_Name": split_proxy_name,
|
||||||
|
"Proxy_Type": split_proxy_type,
|
||||||
|
"Base_Ticker": base_ticker,
|
||||||
|
"Constituent_Code": h["Constituent_Code"],
|
||||||
|
"Constituent_Name": h["Constituent_Name"],
|
||||||
|
"Weight": h["Weight"],
|
||||||
|
"Is_ETF": "N",
|
||||||
|
"Enabled": "Y",
|
||||||
|
"Effective_Date": today,
|
||||||
|
"Source": split_source,
|
||||||
|
"Transport_Mode": "HTML_SERVER_RENDERED",
|
||||||
|
"Source_URL": split_source_url,
|
||||||
|
"Source_AsOf": today,
|
||||||
|
"Sector_Check": spec["sector"],
|
||||||
|
"Weight_Sum_All": weight_sum,
|
||||||
|
"Weight_Sum_Stocks_Only": weight_sum,
|
||||||
|
"ETF_Rows": 0,
|
||||||
|
"Status": "OK",
|
||||||
|
})
|
||||||
|
sector_stats.append({
|
||||||
|
"sector": spec["sector"],
|
||||||
|
"proxy_ticker": split_proxy_ticker,
|
||||||
|
"proxy_name": split_proxy_name,
|
||||||
|
"proxy_type": split_proxy_type,
|
||||||
|
"source_kind": split_source,
|
||||||
|
"transport_mode": "HTML_SERVER_RENDERED",
|
||||||
|
"source_url": split_source_url,
|
||||||
|
"source_asof": today,
|
||||||
|
"constituent_count": len(holdings),
|
||||||
|
"weight_sum": weight_sum,
|
||||||
|
"status": "CURRENT",
|
||||||
|
"refresh_reason": "NAVER_ETF_PAGE_SPLIT",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
except Exception as exc:
|
||||||
|
split_message = str(exc)
|
||||||
|
split_source_kind = "NAVER_ETF_PAGE_FAIL"
|
||||||
|
|
||||||
|
# 실패 시는 투명하게 남기고, 섹터 누락은 그대로 드러낸다.
|
||||||
|
sector_stats.append({
|
||||||
|
"sector": spec["sector"],
|
||||||
|
"proxy_ticker": split_proxy_ticker,
|
||||||
|
"proxy_name": split_proxy_name,
|
||||||
|
"proxy_type": split_proxy_type,
|
||||||
|
"source_kind": split_source_kind,
|
||||||
|
"transport_mode": "LAYOUT_CHANGED" if split_source_kind == "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" else "UNKNOWN",
|
||||||
|
"source_url": split_source_url,
|
||||||
|
"source_asof": today,
|
||||||
|
"constituent_count": 0,
|
||||||
|
"weight_sum": 0.0,
|
||||||
|
"status": "FAIL" if "FAIL" in split_source_kind else "WARN",
|
||||||
|
"refresh_reason": split_message or "split_sector_fallback",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
|
||||||
|
source = "SHEET_INPUT"
|
||||||
|
source_url = ""
|
||||||
|
message = ""
|
||||||
|
source_kind = "SHEET_INPUT"
|
||||||
|
|
||||||
|
if proxy_type != "ETF":
|
||||||
|
source_kind = "REPRESENTATIVE_STOCK_PROXY"
|
||||||
|
source = source_kind
|
||||||
|
source_url = f"{NAVER_BASE}/item/main.naver?code={proxy_ticker}" if proxy_ticker else ""
|
||||||
|
fallback_rows = bundle["rows"][:limit] if bundle["rows"] else []
|
||||||
|
weight_sum = 0.0
|
||||||
|
for row in fallback_rows:
|
||||||
|
weight = row.get("Weight")
|
||||||
|
try:
|
||||||
|
weight_sum += float(weight) if weight not in (None, "") else 0.0
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
refreshed.append({
|
||||||
|
"Sector": sector,
|
||||||
|
"Proxy_Ticker": proxy_ticker,
|
||||||
|
"Proxy_Name": proxy_name,
|
||||||
|
"Proxy_Type": proxy_type,
|
||||||
|
"Base_Ticker": base_ticker,
|
||||||
|
"Constituent_Code": _normalize_code(row.get("Constituent_Code")),
|
||||||
|
"Constituent_Name": _clean_text(row.get("Constituent_Name")),
|
||||||
|
"Weight": float(row.get("Weight") or 0),
|
||||||
|
"Is_ETF": _clean_text(row.get("Is_ETF")) or "N",
|
||||||
|
"Enabled": "Y",
|
||||||
|
"Effective_Date": today,
|
||||||
|
"Source": source_kind,
|
||||||
|
"Transport_Mode": "HTML_SERVER_RENDERED" if source_kind == "REPRESENTATIVE_STOCK_PROXY" else "MANUAL_OR_TEMPLATE",
|
||||||
|
"Source_URL": source_url,
|
||||||
|
"Source_AsOf": today,
|
||||||
|
"Sector_Check": sector,
|
||||||
|
"Weight_Sum_All": weight_sum,
|
||||||
|
"Weight_Sum_Stocks_Only": weight_sum,
|
||||||
|
"ETF_Rows": 0,
|
||||||
|
"Status": "CURRENT",
|
||||||
|
})
|
||||||
|
sector_stats.append({
|
||||||
|
"sector": sector,
|
||||||
|
"proxy_ticker": proxy_ticker,
|
||||||
|
"proxy_name": proxy_name,
|
||||||
|
"proxy_type": proxy_type,
|
||||||
|
"source_kind": source_kind,
|
||||||
|
"transport_mode": "HTML_SERVER_RENDERED" if source_kind == "REPRESENTATIVE_STOCK_PROXY" else "MANUAL_OR_TEMPLATE",
|
||||||
|
"source_url": source_url,
|
||||||
|
"source_asof": today,
|
||||||
|
"constituent_count": len(fallback_rows),
|
||||||
|
"weight_sum": round(weight_sum, 6),
|
||||||
|
"status": "CURRENT",
|
||||||
|
"refresh_reason": "REPRESENTATIVE_STOCK_PROXY",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
|
||||||
|
if proxy_ticker:
|
||||||
|
try:
|
||||||
|
scraped = _parse_naver_etf_holdings(session, proxy_ticker, limit)
|
||||||
|
source_url = scraped.get("source_url", "")
|
||||||
|
source_kind = scraped.get("source_kind", "NAVER_ETF_PAGE_FAIL")
|
||||||
|
holdings = scraped.get("holdings", [])
|
||||||
|
message = scraped.get("message", "")
|
||||||
|
if holdings:
|
||||||
|
source = "NAVER_ETF_PAGE"
|
||||||
|
weight_sum = round(sum(float(h["Weight"]) for h in holdings), 6)
|
||||||
|
for h in holdings:
|
||||||
|
refreshed.append({
|
||||||
|
"Sector": sector,
|
||||||
|
"Proxy_Ticker": proxy_ticker,
|
||||||
|
"Proxy_Name": proxy_name,
|
||||||
|
"Proxy_Type": proxy_type,
|
||||||
|
"Base_Ticker": base_ticker,
|
||||||
|
"Constituent_Code": h["Constituent_Code"],
|
||||||
|
"Constituent_Name": h["Constituent_Name"],
|
||||||
|
"Weight": h["Weight"],
|
||||||
|
"Is_ETF": "N",
|
||||||
|
"Enabled": "Y",
|
||||||
|
"Effective_Date": today,
|
||||||
|
"Source": source,
|
||||||
|
"Transport_Mode": "HTML_SERVER_RENDERED",
|
||||||
|
"Source_URL": source_url,
|
||||||
|
"Source_AsOf": today,
|
||||||
|
"Sector_Check": sector,
|
||||||
|
"Weight_Sum_All": weight_sum,
|
||||||
|
"Weight_Sum_Stocks_Only": weight_sum,
|
||||||
|
"ETF_Rows": 0,
|
||||||
|
"Status": "OK",
|
||||||
|
})
|
||||||
|
sector_stats.append({
|
||||||
|
"sector": sector,
|
||||||
|
"proxy_ticker": proxy_ticker,
|
||||||
|
"proxy_name": proxy_name,
|
||||||
|
"proxy_type": proxy_type,
|
||||||
|
"source_kind": source,
|
||||||
|
"transport_mode": "HTML_SERVER_RENDERED",
|
||||||
|
"source_url": source_url,
|
||||||
|
"source_asof": today,
|
||||||
|
"constituent_count": len(holdings),
|
||||||
|
"weight_sum": weight_sum,
|
||||||
|
"status": "CURRENT",
|
||||||
|
"refresh_reason": "NAVER_ETF_PAGE",
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
except Exception as exc:
|
||||||
|
message = str(exc)
|
||||||
|
source_kind = "NAVER_ETF_PAGE_FAIL"
|
||||||
|
|
||||||
|
# fallback: preserve seed rows but expose the failure transparently
|
||||||
|
fallback_rows = bundle["rows"][:limit] if bundle["rows"] else []
|
||||||
|
weight_sum = 0.0
|
||||||
|
for row in fallback_rows:
|
||||||
|
weight = row.get("Weight")
|
||||||
|
try:
|
||||||
|
weight_sum += float(weight) if weight not in (None, "") else 0.0
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
refreshed.append({
|
||||||
|
"Sector": sector,
|
||||||
|
"Proxy_Ticker": proxy_ticker,
|
||||||
|
"Proxy_Name": proxy_name,
|
||||||
|
"Proxy_Type": proxy_type,
|
||||||
|
"Base_Ticker": base_ticker,
|
||||||
|
"Constituent_Code": _normalize_code(row.get("Constituent_Code")),
|
||||||
|
"Constituent_Name": _clean_text(row.get("Constituent_Name")),
|
||||||
|
"Weight": float(row.get("Weight") or 0),
|
||||||
|
"Is_ETF": _clean_text(row.get("Is_ETF")) or "N",
|
||||||
|
"Enabled": "Y",
|
||||||
|
"Effective_Date": today,
|
||||||
|
"Source": source_kind,
|
||||||
|
"Transport_Mode": "LAYOUT_CHANGED" if source_kind == "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" else "UNKNOWN",
|
||||||
|
"Source_URL": source_url,
|
||||||
|
"Source_AsOf": today,
|
||||||
|
"Sector_Check": sector,
|
||||||
|
"Weight_Sum_All": weight_sum,
|
||||||
|
"Weight_Sum_Stocks_Only": weight_sum,
|
||||||
|
"ETF_Rows": 0,
|
||||||
|
"Status": "FAIL" if source_kind.endswith("FAIL") else "WARN",
|
||||||
|
})
|
||||||
|
sector_stats.append({
|
||||||
|
"sector": sector,
|
||||||
|
"proxy_ticker": proxy_ticker,
|
||||||
|
"proxy_name": proxy_name,
|
||||||
|
"proxy_type": proxy_type,
|
||||||
|
"source_kind": source_kind,
|
||||||
|
"transport_mode": "LAYOUT_CHANGED" if source_kind == "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" else "UNKNOWN",
|
||||||
|
"source_url": source_url,
|
||||||
|
"source_asof": today,
|
||||||
|
"constituent_count": len(fallback_rows),
|
||||||
|
"weight_sum": round(weight_sum, 6),
|
||||||
|
"status": "FAIL" if "FAIL" in source_kind else "WARN",
|
||||||
|
"refresh_reason": message or "seed_fallback",
|
||||||
|
})
|
||||||
|
|
||||||
|
audit_payload = build_sector_universe_refresh_audit({"data": {"sector_universe": refreshed}})
|
||||||
|
return refreshed, {
|
||||||
|
"sector_universe_refresh_audit": audit_payload,
|
||||||
|
"sector_stats": sector_stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _style_title(ws, title: str, subtitle: str) -> None:
|
||||||
|
ws.merge_cells(start_row=1, start_column=1, end_row=1, end_column=max(8, ws.max_column or 8))
|
||||||
|
ws["A1"] = title
|
||||||
|
ws["A1"].font = WHITE_FONT
|
||||||
|
ws["A1"].fill = TITLE_FILL
|
||||||
|
ws["A1"].alignment = Alignment(horizontal="left")
|
||||||
|
ws.merge_cells(start_row=2, start_column=1, end_row=2, end_column=max(8, ws.max_column or 8))
|
||||||
|
ws["A2"] = subtitle
|
||||||
|
ws["A2"].font = NOTE_FONT
|
||||||
|
|
||||||
|
|
||||||
|
def _write_table(ws, start_row: int, start_col: int, headers: list[str], rows: list[list[Any]]) -> int:
|
||||||
|
for i, header in enumerate(headers, start=start_col):
|
||||||
|
cell = ws.cell(start_row, i)
|
||||||
|
cell.value = header
|
||||||
|
cell.font = WHITE_FONT
|
||||||
|
cell.fill = HEADER_FILL
|
||||||
|
cell.alignment = Alignment(horizontal="center")
|
||||||
|
for r_idx, row in enumerate(rows, start=start_row + 1):
|
||||||
|
for c_idx, value in enumerate(row, start=start_col):
|
||||||
|
ws.cell(r_idx, c_idx).value = value
|
||||||
|
return start_row + len(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_sector_universe_sheet(wb, rows: list[dict[str, Any]]) -> None:
|
||||||
|
if "sector_universe" in wb.sheetnames:
|
||||||
|
del wb["sector_universe"]
|
||||||
|
ws = wb.create_sheet("sector_universe")
|
||||||
|
headers = [
|
||||||
|
"Sector", "Proxy_Ticker", "Proxy_Name", "Proxy_Type", "Base_Ticker",
|
||||||
|
"Constituent_Code", "Constituent_Name", "Weight", "Is_ETF", "Enabled",
|
||||||
|
"Effective_Date", "Source", "Transport_Mode", "Source_URL", "Source_AsOf", "Sector_Check",
|
||||||
|
"Weight_Sum_All", "Weight_Sum_Stocks_Only", "ETF_Rows", "Status",
|
||||||
|
]
|
||||||
|
now = _kst_now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
ws["A1"] = f"updated: {now} KST"
|
||||||
|
ws["A1"].font = Font(bold=True)
|
||||||
|
_write_table(ws, 2, 1, headers, [[r.get(h, "") for h in headers] for r in rows])
|
||||||
|
for col_idx, header in enumerate(headers, start=1):
|
||||||
|
if header in {"Proxy_Ticker", "Base_Ticker", "Constituent_Code"}:
|
||||||
|
for r in range(3, ws.max_row + 1):
|
||||||
|
ws.cell(r, col_idx).number_format = "@"
|
||||||
|
if header in {"Weight", "Weight_Sum_All", "Weight_Sum_Stocks_Only"}:
|
||||||
|
for r in range(3, ws.max_row + 1):
|
||||||
|
ws.cell(r, col_idx).number_format = "0.0000"
|
||||||
|
width = 16
|
||||||
|
if header in {"Constituent_Name", "Proxy_Name"}:
|
||||||
|
width = 22
|
||||||
|
elif header in {"Source_URL"}:
|
||||||
|
width = 42
|
||||||
|
elif header in {"Status", "Source", "Sector_Check", "Proxy_Type", "Transport_Mode"}:
|
||||||
|
width = 16
|
||||||
|
ws.column_dimensions[get_column_letter(col_idx)].width = width
|
||||||
|
ws.freeze_panes = "A3"
|
||||||
|
ws.sheet_view.showGridLines = False
|
||||||
|
|
||||||
|
|
||||||
|
def _write_audit_sheet(wb, audit_payload: dict[str, Any]) -> None:
|
||||||
|
audit = audit_payload["sector_universe_refresh_audit"]
|
||||||
|
if "sector_universe_refresh_audit" in wb.sheetnames:
|
||||||
|
del wb["sector_universe_refresh_audit"]
|
||||||
|
ws = wb.create_sheet("sector_universe_refresh_audit")
|
||||||
|
ws.sheet_view.showGridLines = False
|
||||||
|
_style_title(
|
||||||
|
ws,
|
||||||
|
"섹터 월간 갱신 감사",
|
||||||
|
"Naver ETF 페이지 기반 월간 갱신 상태와 provenance 분리 현황을 점검한다.",
|
||||||
|
)
|
||||||
|
summary = audit.get("summary", {})
|
||||||
|
summary_rows = [
|
||||||
|
["formula_id", audit.get("formula_id", "")],
|
||||||
|
["gate", audit.get("gate", "")],
|
||||||
|
["sector_count", summary.get("sector_count", 0)],
|
||||||
|
["current_count", summary.get("current_count", 0)],
|
||||||
|
["due_count", summary.get("due_count", 0)],
|
||||||
|
["overdue_count", summary.get("overdue_count", 0)],
|
||||||
|
["missing_count", summary.get("missing_count", 0)],
|
||||||
|
["template_count", summary.get("template_count", 0)],
|
||||||
|
["sheet_input_count", summary.get("sheet_input_count", 0)],
|
||||||
|
["naver_source_count", summary.get("naver_source_count", 0)],
|
||||||
|
["missing_source_url_count", summary.get("missing_source_url_count", 0)],
|
||||||
|
["stale_sector_count", summary.get("stale_sector_count", 0)],
|
||||||
|
["oldest_source_asof", summary.get("oldest_source_asof", "")],
|
||||||
|
["newest_source_asof", summary.get("newest_source_asof", "")],
|
||||||
|
]
|
||||||
|
_write_table(ws, 4, 1, ["key", "value"], summary_rows)
|
||||||
|
rows = audit.get("rows", []) or []
|
||||||
|
if rows:
|
||||||
|
headers = [
|
||||||
|
"sector", "proxy_ticker", "proxy_name", "proxy_type", "source_kind",
|
||||||
|
"source_url", "source_asof", "age_days", "constituent_count",
|
||||||
|
"stock_count", "etf_count", "weight_sum", "status", "refresh_reason",
|
||||||
|
]
|
||||||
|
_write_table(ws, 4, 4, headers, [[r.get(h, "") for h in headers] for r in rows])
|
||||||
|
for idx, header in enumerate(headers, start=4):
|
||||||
|
width = 16
|
||||||
|
if header in {"sector", "proxy_name", "refresh_reason"}:
|
||||||
|
width = 20
|
||||||
|
elif header == "source_url":
|
||||||
|
width = 42
|
||||||
|
ws.column_dimensions[get_column_letter(idx)].width = width
|
||||||
|
ws.freeze_panes = "A5"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--input", default=str(DEFAULT_INPUT_XLSX))
|
||||||
|
ap.add_argument("--output", default=str(DEFAULT_OUTPUT_XLSX))
|
||||||
|
ap.add_argument("--limit", type=int, default=10, help="Per-sector holdings limit from Naver ETF pages")
|
||||||
|
ap.add_argument("--apply", action="store_true", help="Overwrite the input workbook in place as well")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
input_path = Path(args.input)
|
||||||
|
output_path = Path(args.output)
|
||||||
|
if not input_path.exists():
|
||||||
|
raise FileNotFoundError(input_path)
|
||||||
|
|
||||||
|
wb = load_workbook(input_path)
|
||||||
|
if "sector_universe" not in wb.sheetnames:
|
||||||
|
raise RuntimeError("sector_universe sheet not found")
|
||||||
|
seed_ws = wb["sector_universe"]
|
||||||
|
seed_rows = _extract_sector_seed_rows(seed_ws)
|
||||||
|
refreshed_rows, audit_payload = _build_refreshed_rows(seed_rows, max(1, args.limit))
|
||||||
|
|
||||||
|
_write_sector_universe_sheet(wb, refreshed_rows)
|
||||||
|
_write_audit_sheet(wb, audit_payload)
|
||||||
|
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
wb.save(output_path)
|
||||||
|
if args.apply and input_path.resolve() != output_path.resolve():
|
||||||
|
shutil.copy2(output_path, input_path)
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"status": "OK",
|
||||||
|
"input": str(input_path),
|
||||||
|
"output": str(output_path),
|
||||||
|
"rows": len(refreshed_rows),
|
||||||
|
"sectors": len(audit_payload["sector_stats"]),
|
||||||
|
"current_count": audit_payload["sector_universe_refresh_audit"]["summary"]["current_count"],
|
||||||
|
"overdue_count": audit_payload["sector_universe_refresh_audit"]["summary"]["overdue_count"],
|
||||||
|
"template_count": audit_payload["sector_universe_refresh_audit"]["summary"]["template_count"],
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from openpyxl import load_workbook
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DEFAULT_XLSX = ROOT / "GatherTradingData.xlsx"
|
||||||
|
MAX_AGE_DAYS = 31
|
||||||
|
|
||||||
|
|
||||||
|
def _txt(value: Any, default: str = "") -> str:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, str):
|
||||||
|
return value.strip() or default
|
||||||
|
return str(value).strip() or default
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: Any) -> dt.date | None:
|
||||||
|
text = _txt(value)
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
for fmt in ("%Y-%m-%d", "%Y.%m.%d", "%Y/%m/%d"):
|
||||||
|
try:
|
||||||
|
return dt.datetime.strptime(text[:10], fmt).date()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
return dt.date.fromisoformat(text[:10])
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _age_days(value: Any) -> int | None:
|
||||||
|
parsed = _parse_date(value)
|
||||||
|
if parsed is None:
|
||||||
|
return None
|
||||||
|
today = dt.datetime.now(dt.timezone(dt.timedelta(hours=9))).date()
|
||||||
|
return (today - parsed).days
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--xlsx", default=str(DEFAULT_XLSX))
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
xlsx = Path(args.xlsx)
|
||||||
|
if not xlsx.exists():
|
||||||
|
print(f"[오류] 워크북 없음: {xlsx}")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
wb = load_workbook(xlsx, data_only=True)
|
||||||
|
if "sector_universe" not in wb.sheetnames:
|
||||||
|
print("[FAIL] sector_universe 시트가 없습니다.")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
ws = wb["sector_universe"]
|
||||||
|
headers = [_txt(ws.cell(2, c).value) for c in range(1, ws.max_column + 1)]
|
||||||
|
idx = {name: i for i, name in enumerate(headers) if name}
|
||||||
|
required = ["Sector", "Proxy_Ticker", "Constituent_Code", "Weight", "Source", "Source_URL", "Source_AsOf"]
|
||||||
|
missing_headers = [h for h in required if h not in idx]
|
||||||
|
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
for r in range(3, ws.max_row + 1):
|
||||||
|
row = {h: ws.cell(r, c + 1).value for c, h in enumerate(headers) if h}
|
||||||
|
if not any(v not in (None, "") for v in row.values()):
|
||||||
|
continue
|
||||||
|
rows.append(row)
|
||||||
|
|
||||||
|
sector_map: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for row in rows:
|
||||||
|
sector = _txt(row.get("Sector"))
|
||||||
|
if sector:
|
||||||
|
sector_map.setdefault(sector, []).append(row)
|
||||||
|
|
||||||
|
template_rows = 0
|
||||||
|
representative_rows = 0
|
||||||
|
sheet_input_rows = 0
|
||||||
|
naver_rows = 0
|
||||||
|
layout_changed_rows = 0
|
||||||
|
fail_rows = 0
|
||||||
|
missing_source_url = 0
|
||||||
|
stale_rows = 0
|
||||||
|
mixed_sector_count = 0
|
||||||
|
sector_status_rows: list[str] = []
|
||||||
|
|
||||||
|
for sector, sector_rows in sector_map.items():
|
||||||
|
source_kinds = {_txt(r.get("Source"), "SHEET_INPUT") or "SHEET_INPUT" for r in sector_rows}
|
||||||
|
if len(source_kinds) > 1:
|
||||||
|
mixed_sector_count += 1
|
||||||
|
sector_status_rows.append(f"{sector}:MIXED({','.join(sorted(source_kinds))})")
|
||||||
|
|
||||||
|
sector_template = any(src == "DEFAULT_TEMPLATE" for src in source_kinds)
|
||||||
|
sector_rep = any(src == "REPRESENTATIVE_STOCK_PROXY" for src in source_kinds)
|
||||||
|
sector_input = any(src == "SHEET_INPUT" for src in source_kinds)
|
||||||
|
sector_naver = any(src == "NAVER_ETF_PAGE" for src in source_kinds)
|
||||||
|
sector_layout_changed = any(src == "NAVER_ETF_PAGE_FAIL_LAYOUT_CHANGED" for src in source_kinds)
|
||||||
|
sector_fail = any("FAIL" in src for src in source_kinds)
|
||||||
|
|
||||||
|
if sector_template:
|
||||||
|
template_rows += len(sector_rows)
|
||||||
|
if sector_rep:
|
||||||
|
representative_rows += len(sector_rows)
|
||||||
|
if sector_input:
|
||||||
|
sheet_input_rows += len(sector_rows)
|
||||||
|
if sector_naver:
|
||||||
|
naver_rows += len(sector_rows)
|
||||||
|
if sector_layout_changed:
|
||||||
|
layout_changed_rows += len(sector_rows)
|
||||||
|
if sector_fail:
|
||||||
|
fail_rows += len(sector_rows)
|
||||||
|
|
||||||
|
source_urls = {_txt(r.get("Source_URL")) for r in sector_rows if _txt(r.get("Source_URL"))}
|
||||||
|
if not source_urls:
|
||||||
|
missing_source_url += len(sector_rows)
|
||||||
|
|
||||||
|
ages = [_age_days(r.get("Source_AsOf")) for r in sector_rows]
|
||||||
|
age_vals = [a for a in ages if a is not None]
|
||||||
|
if age_vals and max(age_vals) > MAX_AGE_DAYS:
|
||||||
|
stale_rows += sum(1 for a in age_vals if a is not None and a > MAX_AGE_DAYS)
|
||||||
|
sector_status_rows.append(f"{sector}:STALE(max={max(age_vals)})")
|
||||||
|
|
||||||
|
gate = "PASS"
|
||||||
|
if missing_headers:
|
||||||
|
gate = "FAIL"
|
||||||
|
elif template_rows > 0 or fail_rows > 0 or stale_rows > 0 or mixed_sector_count > 0:
|
||||||
|
gate = "FAIL"
|
||||||
|
elif sheet_input_rows > 0:
|
||||||
|
gate = "WARN"
|
||||||
|
|
||||||
|
print(f"[sector_universe_refresh] gate={gate}")
|
||||||
|
print(f" rows={len(rows)} sectors={len(sector_map)}")
|
||||||
|
print(f" naver_rows={naver_rows} representative_rows={representative_rows} layout_changed_rows={layout_changed_rows} sheet_input_rows={sheet_input_rows} template_rows={template_rows} fail_rows={fail_rows}")
|
||||||
|
print(f" missing_source_url={missing_source_url} stale_rows={stale_rows} mixed_sector_count={mixed_sector_count}")
|
||||||
|
if missing_headers:
|
||||||
|
print(f" missing_headers={missing_headers}")
|
||||||
|
if sector_status_rows:
|
||||||
|
print(" sector_flags=" + ", ".join(sector_status_rows[:20]))
|
||||||
|
|
||||||
|
result = {
|
||||||
|
"validator": "validate_sector_universe_monthly_refresh_v1",
|
||||||
|
"gate": gate,
|
||||||
|
"total_rows": len(rows),
|
||||||
|
"sector_count": len(sector_map),
|
||||||
|
"naver_rows": naver_rows,
|
||||||
|
"representative_rows": representative_rows,
|
||||||
|
"layout_changed_rows": layout_changed_rows,
|
||||||
|
"sheet_input_rows": sheet_input_rows,
|
||||||
|
"template_rows": template_rows,
|
||||||
|
"fail_rows": fail_rows,
|
||||||
|
"missing_source_url": missing_source_url,
|
||||||
|
"stale_rows": stale_rows,
|
||||||
|
"mixed_sector_count": mixed_sector_count,
|
||||||
|
"missing_headers": missing_headers,
|
||||||
|
"sector_flags": sector_status_rows,
|
||||||
|
"max_age_days": MAX_AGE_DAYS,
|
||||||
|
}
|
||||||
|
out = ROOT / "Temp" / "sector_universe_refresh_validation.json"
|
||||||
|
out.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
print(f"OUTPUT: {out}")
|
||||||
|
return 0 if gate in {"PASS", "WARN"} else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in New Issue
Block a user