Perplexity

# Perplexity 계산 예제
import torch
import math
from transformers import GPT2LMHeadModel, GPT2Tokenizer

# 1. 모델 및 토크나이저 로드
model = GPT2LMHeadModel.from_pretrained("gpt2")
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model.eval()

def calculate_perplexity(text, model, tokenizer):
    """텍스트의 Perplexity 계산"""
    encodings = tokenizer(text, return_tensors="pt")
    input_ids = encodings.input_ids

    with torch.no_grad():
        outputs = model(input_ids, labels=input_ids)
        # Cross-Entropy Loss
        loss = outputs.loss
        # Perplexity = e^(loss)
        perplexity = math.exp(loss.item())

    return perplexity

# 2. 예제 텍스트로 PPL 계산
text1 = "The quick brown fox jumps over the lazy dog."
text2 = "Purple elephants dance quantum mechanics pizza."

ppl1 = calculate_perplexity(text1, model, tokenizer)
ppl2 = calculate_perplexity(text2, model, tokenizer)

print(f"자연스러운 문장 PPL: {ppl1:.2f}")  # 낮음
print(f"비자연스러운 문장 PPL: {ppl2:.2f}")  # 높음

# 3. Hugging Face evaluate 라이브러리 사용
from evaluate import load
perplexity_metric = load("perplexity", module_type="metric")

# 데이터셋 전체 PPL 계산
texts = ["Hello world.", "AI is transforming the world."]
results = perplexity_metric.compute(
    predictions=texts,
    model_id="gpt2"
)
print(f"평균 PPL: {results['mean_perplexity']:.2f}")