실습 4 — 검색 품질 재기

import json
import numpy as np
import pandas as pd
from sentence_transformers import SentenceTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

text = open("it_helpdesk.md", encoding="utf-8").read()
chunks = ["## " + s.strip() for s in text.split("\n## ")[1:]]
qa = [json.loads(line) for line in open("it_helpdesk_qa.jsonl", encoding="utf-8")]
emb = SentenceTransformer("intfloat/multilingual-e5-small", device="cpu")
E = emb.encode(["passage: " + c for c in chunks], normalize_embeddings=True)
vec = TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 3))
X = vec.fit_transform(chunks)
print("질문 수:", len(qa), "· 유형:", pd.Series([r["유형"] for r in qa]).value_counts().to_dict())
print(qa[2])

# 질문을 한꺼번에 임베딩한다(하나씩 할 때보다 빠르다)
Q = emb.encode(["query: " + r["질문"] for r in qa], normalize_embeddings=True)
sem_scores = Q @ E.T                                          # (질문 수, 조각 수)
kw_scores = cosine_similarity(vec.transform([r["질문"] for r in qa]), X)

def rank_of_gold(scores, row):
    """정답 조각(근거 글이 든 조각)의 순위. 1 부터 센다"""
    order = np.argsort(-scores).tolist()
    gold = [i for i, c in enumerate(chunks) if row["근거"] in c]
    return min(order.index(g) for g in gold) + 1

rows = []
for n, r in enumerate(qa):
    rows.append({"질문": r["질문"], "유형": r["유형"],
                 "키워드": rank_of_gold(kw_scores[n], r), "의미": rank_of_gold(sem_scores[n], r)})
ranks = pd.DataFrame(rows)
print(ranks.head(6).to_string(index=False))

def metrics(r):
    """순위 열 하나로 세 지표를 계산한다"""
    return pd.Series({"hit@1": (r == 1).mean(), "hit@3": (r <= 3).mean(), "MRR": (1 / r).mean()}).round(3)

table = pd.DataFrame({"키워드": metrics(ranks["키워드"]), "의미": metrics(ranks["의미"])})
print(table)
# 유형별 hit@3
print(ranks.groupby("유형")[["키워드", "의미"]].agg(lambda r: (r <= 3).mean()).round(3))

# 한쪽은 1등인데 다른 쪽은 2등 아래인 질문
diff = ranks[(ranks["키워드"] == 1) != (ranks["의미"] == 1)]
print(diff.to_string(index=False))