#!/usr/bin/env python3
"""
B1 数据清洗适配器：2026-06-08 技术方案统计表
复用 clean_methods.py 核心逻辑，替换 DATA_DATE
"""
import sys
sys.path.insert(0, "/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/dashboard/src")

# 动态修改 clean_methods 的全局变量
import importlib.util
spec = importlib.util.spec_from_file_location(
    "clean_methods",
    "/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/dashboard/src/clean_methods.py"
)

# 由于 clean_methods.py 太长且有执行逻辑，直接复制脚本核心到此处
# 采用更可靠的方式：修改源文件中的 DATA_DATE 然后执行

import pandas as pd
import openpyxl
from datetime import datetime, date
from pathlib import Path
import json
import warnings
warnings.filterwarnings('ignore')

# ============================================================
# CONFIG - 适配 2026-06-08
# ============================================================
DATA_DATE = "2026-06-08"
RAW_DIR = Path(f"/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/raw/月报数据/{DATA_DATE}")
OUT_DIR = Path(f"/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/dashboard/data/{DATA_DATE}/cleaned")
OUT_DIR.mkdir(parents=True, exist_ok=True)

INPUT_FILE = RAW_DIR / "技术方案统计表.xlsx"
TODAY = date.today()

# ============================================================
# Step 1: 读取原始数据
# ============================================================
print("📖 读取原始数据...")
wb = openpyxl.load_workbook(INPUT_FILE, data_only=True)
ws = wb['sheet1']

# 提取表头 (Row 2)
headers = {}
for cell in ws[2]:
    if cell.value:
        headers[cell.column_letter] = str(cell.value).strip()

print(f"  表头列: {len(headers)} 列")
for k, v in sorted(headers.items()):
    print(f"    [{k}] {v}")

# 提取数据行 (Row 3+)
raw_rows = []
for row in ws.iter_rows(min_row=3, max_row=ws.max_row, values_only=False):
    vals = {}
    for cell in row:
        try:
            cl = cell.column_letter
            if cl in headers:
                vals[headers[cl]] = cell.value
        except AttributeError:
            pass
    if any(v is not None for v in vals.values()):
        raw_rows.append(vals)

wb.close()
print(f"  原始行数: {len(raw_rows)}")

# ============================================================
# Step 2: 转为 DataFrame
# ============================================================
df = pd.DataFrame(raw_rows)
print(f"\n📊 原始列: {list(df.columns)}")
print(f"  总行数: {len(df)}")

# ============================================================
# Step 3: Forward-fill
# ============================================================
proj_col = '项目名称'
if proj_col in df.columns:
    before = df[proj_col].notna().sum()
    df[proj_col] = df[proj_col].ffill()
    after = df[proj_col].notna().sum()
    print(f"🔧 Forward-fill 项目名称: {before} → {after}")

region_col = '所属区域'
if region_col in df.columns:
    before = df[region_col].notna().sum()
    df[region_col] = df[region_col].ffill()
    after = df[region_col].notna().sum()
    print(f"🔧 Forward-fill 区域列: {before} → {after}")

country_col = '所属国别'
if country_col in df.columns:
    before = df[country_col].notna().sum()
    df[country_col] = df[country_col].ffill()
    after = df[country_col].notna().sum()
    print(f"🔧 Forward-fill 国别列: {before} → {after}")

# 方案名称 forward-fill
scheme_col = '方案名称'
if scheme_col in df.columns:
    before = df[scheme_col].notna().sum()
    df[scheme_col] = df[scheme_col].ffill()
    after = df[scheme_col].notna().sum()
    print(f"🔧 Forward-fill 方案名称: {before} → {after}")

# 是否超一定规模 forward-fill
oversize_col = '是否超一定规模'
if oversize_col in df.columns:
    before = df[oversize_col].notna().sum()
    df[oversize_col] = df[oversize_col].ffill()
    after = df[oversize_col].notna().sum()
    print(f"🔧 Forward-fill 是否超一定规模: {before} → {after}")

# ============================================================
# Step 4: 过滤中东区域
# ============================================================
print(f"\n🔍 区域分布:")
if region_col in df.columns:
    region_counts = df[region_col].value_counts()
    for region, count in region_counts.head(10).items():
        print(f"  {region}: {count}")

ME_REGION = '中国港湾中东区域公司'
if region_col in df.columns:
    df_me = df[df[region_col] == ME_REGION].copy()
else:
    df_me = df.copy()
    print("  ⚠️ 未找到区域列")

print(f"  中东区域行数: {len(df_me)}")

# ============================================================
# Step 5: 派生计算列
# ============================================================
print(f"\n🔧 派生计算列...")

# 5.1 解析日期
date_col = '分部分项工程计划开工日期'
if date_col in df_me.columns:
    df_me[date_col] = pd.to_datetime(df_me[date_col], errors='coerce')

# 5.2 方案状态映射
status_col = None
for col in df_me.columns:
    if '方案状态' in str(col):
        status_col = col
        break

if status_col:
    df_me['方案状态_raw'] = df_me[status_col].astype(str)
else:
    df_me['方案状态_raw'] = '未知'

# 从 W 列重新读取状态
print("  从原始 W 列读取方案状态...")
wb2 = openpyxl.load_workbook(INPUT_FILE, data_only=True)
ws2 = wb2['sheet1']
status_map = {}
for row in ws2.iter_rows(min_row=3, max_row=ws2.max_row, values_only=False):
    seq = row[0].value
    w_val = row[22].value if len(row) > 22 else None  # W列=23列(0-indexed)
    if seq is not None and w_val is not None:
        status_map[int(seq)] = str(w_val).strip()
wb2.close()

if '序号' in df_me.columns:
    df_me['方案状态'] = df_me['序号'].map(status_map).fillna('未知')
    print(f"  映射了 {sum(df_me['方案状态'] != '未知')} 条状态")

# 5.3 是否完成审批
approved_keywords = ['已审批', '已备案']
df_me['是否完成审批'] = df_me['方案状态'].apply(
    lambda x: any(kw in str(x) for kw in approved_keywords)
)

# 5.4 开工年份/月份
if date_col in df_me.columns:
    df_me['开工年份'] = df_me[date_col].dt.year
    df_me['开工月份'] = df_me[date_col].dt.month
    df_me['计划开工日期倒数'] = (df_me[date_col] - pd.Timestamp(TODAY)).dt.days

# 5.5 是否有效登记
def is_valid_registration(row):
    status = str(row.get('方案状态', ''))
    if '已作废' in status:
        return False
    name = str(row.get('方案名称', ''))
    if '作废' in name or '拟作废' in name:
        return False
    return True

df_me['是否有效登记'] = df_me.apply(is_valid_registration, axis=1)

# 5.6 预警信号
def calc_warning(row):
    status = str(row.get('方案状态', ''))
    days_to_start = row.get('计划开工日期倒数', 999)
    is_approved = row.get('是否完成审批', False)
    is_valid = row.get('是否有效登记', True)
    
    if not is_valid:
        return '无预警信号'
    if is_approved:
        return '无预警信号'
    
    if '在实施' in status and ('未审批' in status or '审批中' in status):
        return '🔴 红色预警'
    
    if days_to_start is None or pd.isna(days_to_start):
        return '无预警信号'
    
    if days_to_start <= 30 and days_to_start >= 0:
        return '🟠 橙色预警'
    elif days_to_start <= 45 and days_to_start > 30:
        return '🟡 黄色预警'
    
    return '无预警信号'

df_me['预警信号'] = df_me.apply(calc_warning, axis=1)

# 5.7 审批进行中
df_me['审批进行中'] = df_me['方案状态'].apply(
    lambda x: '在实施' in str(x) or '审批中' in str(x)
)

# 5.8 是否逾期未审批
if date_col in df_me.columns:
    df_me['是否逾期未审批'] = (
        (df_me[date_col] < pd.Timestamp(TODAY)) &
        (~df_me['是否完成审批'])
    )
else:
    df_me['是否逾期未审批'] = False

# ============================================================
# Step 6: 汇总统计
# ============================================================
total = len(df_me)
valid_df = df_me[df_me['是否有效登记']]
oversized = (valid_df['是否超一定规模'].astype(str) == '是').sum()
approved = valid_df['是否完成审批'].sum()
unapproved = len(valid_df) - approved
warnings_count = (valid_df['预警信号'] != '无预警信号').sum()
red_warnings = (valid_df['预警信号'] == '🔴 红色预警').sum()
orange_warnings = (valid_df['预警信号'] == '🟠 橙色预警').sum()
countries = df_me['所属国别'].nunique() if '所属国别' in df_me.columns else 0
projects = df_me['项目名称'].nunique() if '项目名称' in df_me.columns else 0

print(f"\n📊 汇总统计:")
print(f"  有效方案总数: {len(valid_df)} (总{total}含作废)")
print(f"  超一定规模: {oversized}")
print(f"  已审批: {approved} ({approved/len(valid_df)*100:.0f}%)" if len(valid_df) > 0 else "  已审批: 0")
print(f"  未审批: {unapproved}")
print(f"  预警信号: {warnings_count} (🔴{red_warnings} 🟠{orange_warnings})")
print(f"  覆盖国别: {countries}")
print(f"  覆盖项目: {projects}")

# 状态分布
print(f"\n  方案状态分布:")
for status, count in df_me['方案状态'].value_counts().head(15).items():
    print(f"    {status}: {count}")

# ============================================================
# Step 7: 输出
# ============================================================
PARQUET_PATH = OUT_DIR / "methods_cleaned.parquet"
df_me.to_parquet(PARQUET_PATH, index=False)
print(f"\n✅ 清洗后数据: {PARQUET_PATH} ({len(df_me)} 行 × {len(df_me.columns)} 列)")

CSV_PATH = OUT_DIR / "methods_cleaned.csv"
df_me.to_csv(CSV_PATH, index=False, encoding='utf-8-sig')
print(f"✅ CSV审核文件: {CSV_PATH}")

# 校验报告
report = {
    "date": DATA_DATE,
    "source": str(INPUT_FILE),
    "total_rows_raw": len(df),
    "me_rows": len(df_me),
    "summary": {
        "有效方案总数": int(len(valid_df)),
        "超一定规模": int(oversized),
        "已审批": int(approved),
        "审批率": f"{approved/len(valid_df)*100:.0f}%" if len(valid_df) > 0 else "N/A",
        "未审批": int(unapproved),
        "预警信号总数": int(warnings_count),
        "红色预警": int(red_warnings),
        "橙色预警": int(orange_warnings),
        "覆盖国别": int(countries),
        "覆盖项目": int(projects),
    },
    "columns": list(df_me.columns),
    "column_count": len(df_me.columns),
}
with open(OUT_DIR / "validation_report.json", 'w', encoding='utf-8') as f:
    json.dump(report, f, ensure_ascii=False, indent=2)
print(f"✅ 校验报告: {OUT_DIR / 'validation_report.json'}")

# 按项目汇总
proj_summary = valid_df.groupby('项目名称').agg(
    方案总数=('方案名称', 'count'),
    已审批数=('是否完成审批', 'sum'),
    超规数=('是否超一定规模', lambda x: (x.astype(str) == '是').sum()),
    红色预警=('预警信号', lambda x: (x == '🔴 红色预警').sum()),
    橙色预警=('预警信号', lambda x: (x == '🟠 橙色预警').sum()),
).reset_index()
proj_summary['审批率'] = (proj_summary['已审批数'] / proj_summary['方案总数'] * 100).round(0).astype(int).astype(str) + '%'

# 加入国别
if '所属国别' in df_me.columns:
    country_map = valid_df.groupby('项目名称')['所属国别'].first().to_dict()
    proj_summary['国别'] = proj_summary['项目名称'].map(country_map)

proj_summary = proj_summary.sort_values('方案总数', ascending=False)

PROJ_SUMMARY_PATH = OUT_DIR / "project_summary.csv"
proj_summary.to_csv(PROJ_SUMMARY_PATH, index=False, encoding='utf-8-sig')
print(f"✅ 项目汇总: {PROJ_SUMMARY_PATH} ({len(proj_summary)} 项目)\n")

print("📋 项目汇总预览:")
print(proj_summary.head(20).to_string())
