Constitutional AI

# Constitutional AI 개념적 구현 (의사코드)
# 실제 CAI 훈련은 대규모 컴퓨팅 자원이 필요합니다

class ConstitutionalAI:
    def __init__(self, base_model, principles: list[str]):
        self.model = base_model
        # 헌법 원칙 예시
        self.constitution = principles or [
            "Choose the response that is most helpful while being safe.",
            "Choose the response that sounds most similar to what a wise, ethical expert would say.",
            "Choose the response that is least likely to cause harm.",
        ]

    def critique_and_revise(self, prompt: str, initial_response: str) -> str:
        """1단계: 자기 비평 및 수정"""
        critique_prompt = f"""
        Human: {prompt}
        Assistant: {initial_response}

        Critique Request: Identify specific ways in which the assistant's
        response is harmful, unethical, or unhelpful based on these principles:
        {self.constitution}

        Critique: """

        critique = self.model.generate(critique_prompt)

        revision_prompt = f"""
        Based on the critique above, please rewrite the response to be
        more helpful, harmless, and honest while still addressing the
        human's question.

        Revision: """

        revised_response = self.model.generate(critique_prompt + critique + revision_prompt)
        return revised_response

    def preference_ranking(self, prompt: str, responses: list[str]) -> int:
        """2단계: RLAIF를 위한 선호도 평가"""
        ranking_prompt = f"""
        Consider the following responses to: "{prompt}"

        Response A: {responses[0]}
        Response B: {responses[1]}

        Which response better follows these principles?
        {self.constitution}

        The better response is: """

        # AI가 헌법에 더 부합하는 응답을 선택
        preference = self.model.generate(ranking_prompt)
        return 0 if "A" in preference else 1

# 사용 예시
principles = [
    "항상 정확하고 검증 가능한 정보를 제공한다",
    "유해하거나 위험한 행동을 조장하지 않는다",
    "사용자의 프라이버시를 존중한다"
]

cai = ConstitutionalAI(base_model, principles)
revised = cai.critique_and_revise(user_prompt, initial_output)