# pylint: disable=line-too-long,useless-suppression
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------

"""
DESCRIPTION:
    Given an AIProjectClient, this sample demonstrates how to use the synchronous
    `openai.evals.*` methods to create, get and list evaluation and eval runs
    using a CSV file uploaded as a dataset.

    Unlike JSONL-based evaluations, this sample uses the `csv` data source type
    to run evaluations directly against a CSV file.

USAGE:
    python sample_evaluations_builtin_with_csv.py

    Before running the sample:

    pip install "azure-ai-projects>=2.0.0" python-dotenv

    Set these environment variables with your own values:
    1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your
       Microsoft Foundry project. It has the form: https://<account_name>.services.ai.azure.com/api/projects/<project_name>.
    2) FOUNDRY_MODEL_NAME - Required. The name of the model deployment to use for evaluation.
    3) DATASET_NAME - Optional. The name of the Dataset to create and use in this sample.
    4) DATASET_VERSION - Optional. The version of the Dataset to create and use in this sample.
    5) DATA_FOLDER - Optional. The folder path where the data files for upload are located.
"""

import os
import time
from datetime import datetime, timezone
from pprint import pprint

from dotenv import load_dotenv
from openai.types.eval_create_params import DataSourceConfigCustom

from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import (
    DatasetVersion,
    EvalCsvFileIdSource,
    EvalCsvRunDataSource,
    TestingCriterionAzureAIEvaluator,
)

load_dotenv()


endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
model_deployment_name = os.environ.get("FOUNDRY_MODEL_NAME", "")
dataset_name = os.environ.get("DATASET_NAME", "")
dataset_version = os.environ.get("DATASET_VERSION", "1")

# Construct the paths to the data folder and CSV data file used in this sample
script_dir = os.path.dirname(os.path.abspath(__file__))
data_folder = os.environ.get("DATA_FOLDER", os.path.join(script_dir, "data_folder"))
data_file = os.path.join(data_folder, "sample_data_evaluation.csv")

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
    project_client.get_openai_client() as client,
):

    print("Upload a CSV file and create a new Dataset to reference the file.")
    dataset: DatasetVersion = project_client.datasets.upload_file(
        name=dataset_name or f"eval-csv-data-{datetime.now(timezone.utc).strftime('%Y-%m-%d_%H%M%S_UTC')}",
        version=dataset_version,
        file_path=data_file,
    )
    pprint(dataset)

    data_source_config = DataSourceConfigCustom(
        type="custom",
        item_schema={
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "response": {"type": "string"},
                "context": {"type": "string"},
                "ground_truth": {"type": "string"},
            },
            "required": [],
        },
        include_sample_schema=True,
    )

    testing_criteria = [
        TestingCriterionAzureAIEvaluator(
            type="azure_ai_evaluator",
            name="violence",
            evaluator_name="builtin.violence",
            data_mapping={"query": "{{item.query}}", "response": "{{item.response}}"},
            initialization_parameters={"model": f"{model_deployment_name}"},
        ),
        TestingCriterionAzureAIEvaluator(type="azure_ai_evaluator", name="f1", evaluator_name="builtin.f1_score"),
        TestingCriterionAzureAIEvaluator(
            type="azure_ai_evaluator",
            name="coherence",
            evaluator_name="builtin.coherence",
            data_mapping={"query": "{{item.query}}", "response": "{{item.response}}"},
            initialization_parameters={"model": f"{model_deployment_name}"},
        ),
    ]

    print("Creating evaluation")
    eval_object = client.evals.create(
        name="CSV evaluation with built-in evaluators",
        data_source_config=data_source_config,
        testing_criteria=testing_criteria,
    )
    print(f"Evaluation created (id: {eval_object.id}, name: {eval_object.name})")

    print("Creating evaluation run with CSV data source")
    eval_run_object = client.evals.runs.create(
        eval_id=eval_object.id,
        name="csv_evaluation_run",
        metadata={"team": "eval-exp", "scenario": "csv-eval-v1"},
        data_source=EvalCsvRunDataSource(
            type="csv",
            source=EvalCsvFileIdSource(
                type="file_id",
                id=dataset.id if dataset.id else "",
            ),
        ),
    )

    print(f"Evaluation run created (id: {eval_run_object.id})")
    pprint(eval_run_object)

    while True:
        run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
        if run.status in ["completed", "failed"]:
            output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
            pprint(output_items)
            print(f"Eval Run Report URL: {run.report_url}")

            break
        time.sleep(5)
        print("Waiting for evaluation run to complete...")

    client.evals.delete(eval_id=eval_object.id)
    print("Evaluation deleted")
