Microsoft Foundry と GitHub Actions で AI 評価を自動化する

from azure.ai.evaluation import (
    evaluate,
    GroundednessEvaluator,
    RelevanceEvaluator,
    CoherenceEvaluator,
    FluencyEvaluator,
    ContentSafetyEvaluator,
)
from azure.identity import DefaultAzureCredential

credential = DefaultAzureCredential()

project_config = {
    "subscription_id": "<AZURE_SUBSCRIPTION_ID>",
    "resource_group_name": "<AZURE_RESOURCE_GROUP>",
    "project_name": "<FOUNDRY_PROJECT_NAME>",
}

evaluators = {
    "groundedness": GroundednessEvaluator(credential=credential, azure_ai_project=project_config),
    "relevance": RelevanceEvaluator(credential=credential, azure_ai_project=project_config),
    "coherence": CoherenceEvaluator(credential=credential, azure_ai_project=project_config),
    "fluency": FluencyEvaluator(credential=credential, azure_ai_project=project_config),
    "safety": ContentSafetyEvaluator(credential=credential, azure_ai_project=project_config),
}

def evaluate_results(metrics: dict) -> dict:
    """しきい値チェックと合否判定"""
    thresholds = {
        "groundedness": 4.0,
        "relevance": 4.0,
        "coherence": 3.5,   # 長文生成は少し緩め
        "fluency": 4.0,
    }

    failed_metrics = []
    warnings = []

    for metric, threshold in thresholds.items():
        score = metrics.get(metric, 0)
        if score < threshold:
            failed_metrics.append(f"{metric}: {score:.2f} (閾値: {threshold})")
        elif score < threshold + 0.3:
            # しきい値は超えたが、マージンが小さい場合は警告
            warnings.append(f"{metric}: {score:.2f} (注意: 閾値に近い)")

    return {
        "passed": len(failed_metrics) == 0,
        "failed_metrics": failed_metrics,
        "warnings": warnings,
    }

{"query": "パスワードをリセットするには？", "context": "パスワードリセットは社内ポータル > アカウント設定 > パスワードリセットから実施できます。", "response": "社内ポータルのアカウント設定からパスワードリセットが可能です。"}
{"query": "VPN が繋がらない場合は？", "context": "VPN 接続エラーの場合は、まず Cisco AnyConnect を再起動してください。改善しない場合は IT ヘルプデスクへ連絡。", "response": "Cisco AnyConnect を再起動してください。それでも接続できない場合は IT ヘルプデスクへご連絡ください。"}
{"query": "在宅勤務の申請手順は？", "context": "在宅勤務申請は Workday の「勤怠管理」メニューから翌週分を金曜 17:00 までに提出が必要。", "response": "Workday の勤怠管理から申請できます。翌週分は金曜 17:00 が締め切りです。"}

import json
import random

def sample_from_logs(log_file: str, n: int = 100, seed: int = 42) -> list[dict]:
    """本番ログから評価データをサンプリングする"""
    random.seed(seed)
    logs = []
    with open(log_file) as f:
        for line in f:
            entry = json.loads(line)
            # PII フィールドを除去（メールアドレス・氏名など）
            entry.pop("user_email", None)
            entry.pop("user_name", None)
            logs.append(entry)
    return random.sample(logs, min(n, len(logs)))

pip install azure-ai-evaluation azure-identity

#!/usr/bin/env python3
"""run_evaluation.py — バッチ評価スクリプト"""
import argparse
import json
import sys
from azure.ai.evaluation import (
    evaluate,
    GroundednessEvaluator,
    RelevanceEvaluator,
    CoherenceEvaluator,
    FluencyEvaluator,
)
from azure.identity import DefaultAzureCredential

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--test-data", required=True, help="JSONL テストデータのパス")
    parser.add_argument("--output", default="results.json", help="結果の出力先")
    parser.add_argument("--threshold-groundedness", type=float, default=4.0)
    parser.add_argument("--threshold-relevance", type=float, default=4.0)
    parser.add_argument("--threshold-coherence", type=float, default=3.5)
    args = parser.parse_args()

    credential = DefaultAzureCredential()
    project_config = {
        "subscription_id": os.environ["AZURE_SUBSCRIPTION_ID"],
        "resource_group_name": os.environ["AZURE_RESOURCE_GROUP"],
        "project_name": os.environ["FOUNDRY_PROJECT_NAME"],
    }

    evaluators = {
        "groundedness": GroundednessEvaluator(
            credential=credential, azure_ai_project=project_config
        ),
        "relevance": RelevanceEvaluator(
            credential=credential, azure_ai_project=project_config
        ),
        "coherence": CoherenceEvaluator(
            credential=credential, azure_ai_project=project_config
        ),
        "fluency": FluencyEvaluator(
            credential=credential, azure_ai_project=project_config
        ),
    }

    print(f"⏳ 評価開始: {args.test_data}")
    raw = evaluate(
        data=args.test_data,
        evaluators=evaluators,
    )

    # メトリックの平均スコアを集計
    metrics = {
        key: raw["metrics"].get(f"groundedness.gpt_groundedness", 0)
        for key in ["groundedness", "relevance", "coherence", "fluency"]
    }
    # 実際のキー名は azure-ai-evaluation のバージョンで異なる場合がある
    metrics = raw.get("metrics", {})

    # しきい値チェック
    thresholds = {
        "groundedness": args.threshold_groundedness,
        "relevance": args.threshold_relevance,
        "coherence": args.threshold_coherence,
    }

    failed = []
    for metric, threshold in thresholds.items():
        score = metrics.get(metric, 0)
        if score < threshold:
            failed.append({"metric": metric, "score": score, "threshold": threshold})

    output = {
        "metrics": metrics,
        "passed": len(failed) == 0,
        "failed_details": failed,
        "total_examples": raw.get("row_count", 0),
        "failed_examples": len(failed),
    }

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(output, f, ensure_ascii=False, indent=2)

    print(f"✅ 評価完了: {args.output}")
    for metric, score in metrics.items():
        status = "✅" if score >= thresholds.get(metric, 0) else "❌"
        print(f"  {status} {metric}: {score:.2f}")

    if not output["passed"]:
        print("\n❌ 品質チェック FAILED")
        sys.exit(1)  # GitHub Actions を失敗にする
    else:
        print("\n✅ 品質チェック PASSED")

if __name__ == "__main__":
    main()

{
  "metrics": {
    "groundedness": 4.25,
    "relevance": 4.10,
    "coherence": 3.85,
    "fluency": 4.30
  },
  "passed": true,
  "failed_details": [],
  "total_examples": 150,
  "failed_examples": 0
}

name: Evaluate Prompt Changes

on:
  pull_request:
    branches: [main]
    paths:
      - 'prompts/**'
      - 'config/**'
      - 'src/agents/**'   # エージェントのコードも評価対象に含める

permissions:
  id-token: write
  contents: read
  pull-requests: write

jobs:
  run-evaluation:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Azure login (キーレス認証)
        uses: azure/login@v2
        with:
          client-id: ${{ vars.AZURE_CLIENT_ID }}
          tenant-id: ${{ vars.AZURE_TENANT_ID }}
          subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}

      - name: Run evaluation script
        run: |
          python run_evaluation.py \
            --test-data test-data/test_dataset.jsonl \
            --output results.json \
            --threshold-groundedness 4.0 \
            --threshold-relevance 4.0 \
            --threshold-coherence 3.5
        env:
          AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
          AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
          FOUNDRY_PROJECT_NAME: ${{ vars.FOUNDRY_PROJECT_NAME }}

      - name: Post results to PR
        if: always()   # 失敗でも必ずコメントを投稿する
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const results = JSON.parse(fs.readFileSync('results.json'));
            
            const statusIcon = results.passed ? '✅ PASSED' : '❌ FAILED';
            
            let metricsTable = '| メトリック | スコア | 閾値 | 判定 |\n|---|---|---|---|\n';
            const thresholds = { groundedness: 4.0, relevance: 4.0, coherence: 3.5, fluency: 4.0 };
            for (const [key, score] of Object.entries(results.metrics)) {
              const threshold = thresholds[key] || '-';
              const icon = threshold !== '-' && score >= threshold ? '✅' : (threshold !== '-' ? '❌' : '—');
              metricsTable += `| ${key} | ${score.toFixed(2)} | ${threshold} | ${icon} |\n`;
            }

            const failedDetails = results.failed_details.length > 0
              ? '\n\n**失敗したメトリック:**\n' + results.failed_details
                  .map(d => `- ${d.metric}: ${d.score.toFixed(2)} (閾値: ${d.threshold})`)
                  .join('\n')
              : '';

            const comment = `## 🤖 AI 評価結果\n\n**ステータス: ${statusIcon}**\n\n${metricsTable}${failedDetails}\n\n評価件数: ${results.total_examples} 件`;
            
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: comment
            });

# アプリ登録の作成
az ad app create --display-name "github-actions-eval"
APP_ID=$(az ad app list --display-name "github-actions-eval" --query "[0].appId" -o tsv)

# フェデレーション資格情報の設定（main ブランチと PR トリガー両方を登録）
az ad app federated-credential create --id $APP_ID --parameters '{
  "name": "github-main",
  "issuer": "https://token.actions.githubusercontent.com",
  "subject": "repo:YOUR_ORG/YOUR_REPO:ref:refs/heads/main",
  "audiences": ["api://AzureADTokenExchange"]
}'

az ad app federated-credential create --id $APP_ID --parameters '{
  "name": "github-pr",
  "issuer": "https://token.actions.githubusercontent.com",
  "subject": "repo:YOUR_ORG/YOUR_REPO:pull_request",
  "audiences": ["api://AzureADTokenExchange"]
}'

# ロールの割り当て
az role assignment create \
  --assignee $APP_ID \
  --role "Cognitive Services User" \
  --scope /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.CognitiveServices/accounts/{foundry}

on:
  schedule:
    - cron: '0 0 * * *'   # 毎日 0:00 UTC（日本時間 9:00）
  workflow_dispatch:        # 手動トリガーも残す