2026-01-22 17:05:35,927 INFO Evaluation progress: 0% - Started 2026-01-22 17:05:35,928 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:05:35,934 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 50, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:05:35,936 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:05:35,937 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:05:35,937 INFO COMPATIBLE 2026-01-22 17:05:35,938 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:05:35,939 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:05:35,940 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:05:35,941 INFO COMPATIBLE 2026-01-22 17:05:35,941 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:05:35,942 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:05:35,943 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:05:35,944 INFO COMPATIBLE 2026-01-22 17:05:35,944 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:05:35,945 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:05:35,946 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:05:35,947 INFO COMPATIBLE 2026-01-22 17:05:35,947 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:05:35,948 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:05:35,949 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:05:35,950 INFO COMPATIBLE 2026-01-22 17:05:35,950 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:05:35,951 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:05:35,952 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:05:35,953 INFO COMPATIBLE 2026-01-22 17:05:35,954 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:05:35,954 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:05:35,955 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:05:35,956 INFO COMPATIBLE 2026-01-22 17:05:35,957 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:05:35,958 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:05:35,958 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:05:35,960 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:05:35,961 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:05:35,966 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:05:36,459 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:05:36,505 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:05:36,572 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:05:36,658 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:05:36,770 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:05:36,880 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:05:36,962 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:05:37,029 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:05:37,102 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:05:37,157 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:05:37,228 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:05:37,300 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:05:37,360 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:05:37,466 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:05:37,524 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:05:37,606 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:05:37,706 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:05:37,813 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:05:37,879 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:05:37,972 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:05:38,059 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:05:38,157 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:05:38,278 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:05:38,399 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:05:38,462 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:05:38,562 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:05:38,629 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:05:38,706 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:05:38,806 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:05:38,872 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:05:38,959 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:05:39,094 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:05:39,192 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:05:39,241 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:05:39,343 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:05:39,446 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:05:39,509 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:05:39,609 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:05:39,682 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:05:39,779 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:05:39,905 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:05:40,009 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:05:40,079 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:05:40,139 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:05:40,220 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:05:40,301 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:05:40,380 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:05:40,482 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:05:40,577 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:05:40,711 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:05:41,129 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:05:41,130 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:05:41,133 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:05:41,134 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:05:41,139 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:05:41,683 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:05:41,745 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:05:41,843 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:05:41,945 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:05:42,079 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:05:42,169 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:05:42,239 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:05:42,299 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:05:42,375 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:05:42,421 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:05:42,485 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:05:42,576 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:05:42,649 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:05:42,803 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:05:42,863 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:05:42,944 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:05:43,071 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:05:43,201 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:05:43,291 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:05:43,398 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:05:43,529 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:05:43,665 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:05:43,759 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:05:43,877 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:05:43,945 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:05:44,079 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:05:44,156 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:05:44,262 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:05:44,417 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:05:44,519 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:05:44,607 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:05:44,779 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:05:44,914 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:05:44,958 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:05:45,095 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:05:45,249 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:05:45,314 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:05:45,454 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:05:45,526 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:05:45,646 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:05:45,777 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:05:45,897 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:05:46,001 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:05:46,086 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:05:46,205 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:05:46,326 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:05:46,433 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:05:46,567 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:05:46,663 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:05:46,846 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:05:47,335 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:05:47,337 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:05:47,341 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:05:47,342 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:05:47,686 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:05:47,687 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:05:47,692 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:05:47,693 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:05:47,699 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:05:48,333 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:05:48,873 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:05:49,455 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:05:49,997 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:05:50,585 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:05:51,121 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:05:51,607 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:05:52,132 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:05:52,634 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:05:53,112 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:05:53,591 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:05:54,088 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:05:54,579 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:05:55,148 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:05:55,640 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:05:56,158 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:05:56,730 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:05:57,295 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:05:57,737 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:05:58,305 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:05:58,898 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:05:59,462 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:06:00,003 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:06:00,526 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:06:01,011 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:06:01,561 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:06:02,149 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:06:02,590 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:06:03,120 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:06:03,661 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:06:04,243 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:06:04,857 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:06:05,421 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:06:05,973 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:06:06,463 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:06:07,055 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:06:07,642 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:06:08,106 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:06:08,688 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:06:09,387 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:06:09,978 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:06:10,545 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:06:11,090 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:06:11,619 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:06:11,999 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:06:12,525 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:06:13,116 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:06:13,716 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:06:14,267 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:06:14,861 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:06:15,734 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:06:15,735 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:06:15,740 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:06:15,741 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:06:15,747 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:06:15,749 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:06:15,751 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:06:15,753 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:06:15,755 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:06:15,756 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:06:15,758 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:06:15,760 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:06:15,761 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:06:15,763 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:06:15,766 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:06:15,767 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:06:15,770 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:06:15,771 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:06:15,773 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:06:15,775 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:06:15,776 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:06:15,779 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:06:15,781 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:06:15,784 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:06:15,787 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:06:15,790 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:06:15,794 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:06:15,797 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:06:15,801 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:06:15,804 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:06:15,807 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:06:15,811 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:06:15,814 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:06:15,816 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:06:15,820 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:06:15,823 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:06:15,825 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:06:15,827 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:06:15,829 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:06:15,832 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:06:15,835 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:06:15,839 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:06:15,842 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:06:15,845 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:06:15,848 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:06:15,851 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:06:15,855 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:06:15,857 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:06:15,861 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:06:15,866 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:06:15,868 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:06:15,870 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:06:15,877 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:06:15,879 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:06:15,883 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:06:15,933 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:06:15,935 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:06:15,939 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:06:15,940 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:06:15,948 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:06:16,526 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:06:17,042 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:06:17,552 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:06:18,015 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:06:18,528 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:06:19,271 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:06:19,992 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:06:20,532 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:06:21,015 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:06:21,471 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:06:22,115 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:06:22,762 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:06:23,340 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:06:23,875 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:06:24,344 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:06:24,840 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:06:25,361 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:06:25,842 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:06:26,247 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:06:26,741 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:06:27,211 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:06:27,689 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:06:28,158 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:06:28,623 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:06:29,080 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:06:29,590 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:06:30,127 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:06:30,527 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:06:30,981 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:06:31,482 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:06:31,991 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:06:32,472 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:06:32,958 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:06:33,491 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:06:33,927 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:06:34,381 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:06:34,925 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:06:35,327 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:06:35,844 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:06:36,289 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:06:36,797 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:06:37,292 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:06:37,786 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:06:38,248 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:06:38,564 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:06:39,039 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:06:39,596 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:06:40,147 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:06:40,634 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:06:41,072 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:06:41,855 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:06:41,856 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:06:41,861 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:06:41,862 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:06:41,867 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:06:42,425 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:06:42,977 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:06:43,479 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:06:43,940 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:06:44,448 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:06:44,928 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:06:45,390 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:06:45,862 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:06:46,325 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:06:46,767 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:06:47,226 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:06:47,667 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:06:48,098 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:06:48,605 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:06:49,083 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:06:49,611 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:06:50,176 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:06:50,698 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:06:51,118 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:06:51,636 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:06:52,156 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:06:52,832 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:06:53,499 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:06:54,080 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:06:54,565 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:06:55,088 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:06:55,644 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:06:56,052 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:06:56,525 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:06:57,059 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:06:57,608 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:06:58,146 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:06:58,654 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:06:59,204 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:06:59,631 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:07:00,075 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:07:00,615 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:07:01,008 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:07:01,521 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:07:01,971 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:07:02,480 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:07:02,972 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:07:03,483 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:07:03,974 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:07:04,314 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:07:04,826 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:07:05,404 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:07:05,966 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:07:06,432 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:07:06,890 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:07:07,747 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:07:07,749 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:07:07,755 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:07:07,756 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:07:07,758 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:07:07,759 INFO Evaluation progress: 100% - DONE 2026-01-22 17:07:07,765 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom4/h2o-sonar/mli_experiment_25269fb8-8cb5-4879-be2f-3570da92adc1/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_6deba14a-31cf-48df-aa08-df28e5c3cc30/global_html_fragment/text_html/explanation.html' 2026-01-22 17:07:07,789 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom4/h2o-sonar/mli_experiment_25269fb8-8cb5-4879-be2f-3570da92adc1/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_6deba14a-31cf-48df-aa08-df28e5c3cc30/global_html_fragment/text_html/explanation.html' 2026-01-22 17:07:07,885 INFO Evaluation progress: 0% - Started 2026-01-22 17:07:07,885 INFO Evaluation progress: 0% - Started 2026-01-22 17:07:07,887 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:07:07,887 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:07:07,893 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 50, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:07:07,893 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 50, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:07:07,895 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:07:07,895 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:07:07,896 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:07:07,896 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:07:07,897 INFO COMPATIBLE 2026-01-22 17:07:07,897 INFO COMPATIBLE 2026-01-22 17:07:07,897 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:07:07,897 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:07:07,898 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:07:07,898 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:07:07,899 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:07:07,899 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:07:07,900 INFO COMPATIBLE 2026-01-22 17:07:07,900 INFO COMPATIBLE 2026-01-22 17:07:07,901 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:07:07,901 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:07:07,902 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:07:07,902 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:07:07,903 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:07:07,903 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:07:07,904 INFO COMPATIBLE 2026-01-22 17:07:07,904 INFO COMPATIBLE 2026-01-22 17:07:07,905 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:07:07,905 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:07:07,906 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:07:07,906 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:07:07,907 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:07:07,907 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:07:07,909 INFO COMPATIBLE 2026-01-22 17:07:07,909 INFO COMPATIBLE 2026-01-22 17:07:07,910 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:07:07,910 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:07:07,911 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:07:07,911 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:07:07,912 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:07:07,912 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:07:07,913 INFO COMPATIBLE 2026-01-22 17:07:07,913 INFO COMPATIBLE 2026-01-22 17:07:07,914 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:07:07,914 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:07:07,915 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:07:07,915 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:07:07,916 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:07:07,916 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:07:07,917 INFO COMPATIBLE 2026-01-22 17:07:07,917 INFO COMPATIBLE 2026-01-22 17:07:07,918 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:07:07,918 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:07:07,919 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:07:07,919 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:07:07,921 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:07:07,921 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:07:07,922 INFO COMPATIBLE 2026-01-22 17:07:07,922 INFO COMPATIBLE 2026-01-22 17:07:07,923 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:07:07,923 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:07:07,924 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:07:07,924 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:07:07,925 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:07:07,925 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:07:07,928 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:07:07,928 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:07:07,930 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:07:07,930 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:07:07,936 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:07:07,936 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:07:08,526 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:07:08,526 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:07:08,589 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:07:08,589 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:07:08,692 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:07:08,692 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:07:08,853 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:07:08,853 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:07:09,041 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:07:09,041 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:07:09,171 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:07:09,171 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:07:09,279 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:07:09,279 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:07:09,354 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:07:09,354 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:07:09,416 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:07:09,416 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:07:09,473 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:07:09,473 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:07:09,538 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:07:09,538 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:07:09,597 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:07:09,597 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:07:09,663 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:07:09,663 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:07:09,784 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:07:09,784 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:07:09,848 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:07:09,848 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:07:09,919 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:07:09,919 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:07:10,010 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:07:10,010 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:07:10,107 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:07:10,107 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:07:10,182 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:07:10,182 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:07:10,314 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:07:10,314 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:07:10,423 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:07:10,423 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:07:10,554 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:07:10,554 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:07:10,651 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:07:10,651 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:07:10,762 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:07:10,762 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:07:10,852 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:07:10,852 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:07:10,973 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:07:10,973 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:07:11,046 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:07:11,046 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:07:11,117 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:07:11,117 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:07:11,242 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:07:11,242 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:07:11,328 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:07:11,328 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:07:11,416 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:07:11,416 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:07:11,544 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:07:11,544 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:07:11,650 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:07:11,650 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:07:11,695 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:07:11,695 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:07:11,788 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:07:11,788 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:07:11,901 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:07:11,901 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:07:11,959 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:07:11,959 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:07:12,050 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:07:12,050 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:07:12,143 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:07:12,143 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:07:12,281 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:07:12,281 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:07:12,428 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:07:12,428 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:07:12,554 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:07:12,554 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:07:12,617 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:07:12,617 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:07:12,716 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:07:12,716 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:07:12,798 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:07:12,798 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:07:12,923 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:07:12,923 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:07:13,022 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:07:13,022 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:07:13,139 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:07:13,139 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:07:13,229 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:07:13,229 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:07:13,353 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:07:13,353 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:07:13,813 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:07:13,813 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:07:13,814 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:07:13,814 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:07:13,817 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:07:13,817 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:07:13,818 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:07:13,818 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:07:13,823 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:07:13,823 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:07:14,429 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:07:14,429 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:07:14,502 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:07:14,502 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:07:14,604 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:07:14,604 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:07:14,703 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:07:14,703 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:07:14,847 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:07:14,847 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:07:14,951 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:07:14,951 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:07:15,044 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:07:15,044 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:07:15,113 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:07:15,113 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:07:15,170 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:07:15,170 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:07:15,220 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:07:15,220 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:07:15,287 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:07:15,287 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:07:15,355 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:07:15,355 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:07:15,434 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:07:15,434 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:07:15,620 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:07:15,620 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:07:15,687 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:07:15,687 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:07:15,761 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:07:15,761 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:07:15,882 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:07:15,882 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:07:16,029 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:07:16,029 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:07:16,116 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:07:16,116 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:07:16,235 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:07:16,235 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:07:16,388 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:07:16,388 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:07:16,558 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:07:16,558 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:07:16,655 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:07:16,655 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:07:16,797 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:07:16,797 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:07:16,916 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:07:16,916 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:07:17,151 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:07:17,151 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:07:17,261 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:07:17,261 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:07:17,389 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:07:17,389 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:07:17,532 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:07:17,532 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:07:17,642 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:07:17,642 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:07:17,723 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:07:17,723 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:07:17,908 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:07:17,908 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:07:18,057 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:07:18,057 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:07:18,107 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:07:18,107 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:07:18,242 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:07:18,242 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:07:18,402 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:07:18,402 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:07:18,467 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:07:18,467 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:07:18,594 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:07:18,594 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:07:18,721 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:07:18,721 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:07:18,875 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:07:18,875 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:07:19,021 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:07:19,021 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:07:19,138 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:07:19,138 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:07:19,217 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:07:19,217 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:07:19,344 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:07:19,344 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:07:19,454 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:07:19,454 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:07:19,625 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:07:19,625 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:07:19,747 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:07:19,747 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:07:19,898 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:07:19,898 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:07:19,991 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:07:19,991 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:07:20,163 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:07:20,163 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:07:20,649 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:07:20,649 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:07:20,651 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:07:20,651 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:07:20,654 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:07:20,654 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:07:20,655 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:07:20,655 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:07:21,004 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:07:21,004 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:07:21,006 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:07:21,006 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:07:21,011 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:07:21,011 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:07:21,013 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:07:21,013 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:07:21,021 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:07:21,021 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:07:21,678 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:07:21,678 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:07:22,147 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:07:22,147 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:07:22,724 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:07:22,724 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:07:23,328 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:07:23,328 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:07:23,912 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:07:23,912 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:07:24,430 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:07:24,430 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:07:25,089 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:07:25,089 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:07:25,684 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:07:25,684 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:07:26,219 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:07:26,219 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:07:26,695 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:07:26,695 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:07:27,238 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:07:27,238 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:07:27,852 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:07:27,852 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:07:28,704 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:07:28,704 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:07:29,499 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:07:29,499 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:07:30,202 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:07:30,202 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:07:30,792 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:07:30,792 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:07:31,343 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:07:31,343 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:07:31,944 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:07:31,944 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:07:32,442 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:07:32,442 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:07:33,004 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:07:33,004 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:07:33,559 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:07:33,559 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:07:34,195 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:07:34,195 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:07:34,816 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:07:34,816 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:07:35,428 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:07:35,428 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:07:36,098 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:07:36,098 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:07:36,798 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:07:36,798 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:07:37,350 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:07:37,350 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:07:37,857 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:07:37,857 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:07:38,456 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:07:38,456 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:07:39,029 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:07:39,029 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:07:39,676 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:07:39,676 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:07:40,272 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:07:40,272 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:07:40,876 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:07:40,876 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:07:41,429 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:07:41,429 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:07:41,936 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:07:41,936 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:07:42,532 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:07:42,532 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:07:43,090 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:07:43,090 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:07:43,597 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:07:43,597 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:07:44,129 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:07:44,129 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:07:44,750 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:07:44,750 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:07:45,360 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:07:45,360 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:07:45,909 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:07:45,909 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:07:46,485 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:07:46,485 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:07:47,074 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:07:47,074 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:07:47,546 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:07:47,546 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:07:48,087 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:07:48,087 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:07:48,688 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:07:48,688 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:07:49,331 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:07:49,331 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:07:49,941 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:07:49,941 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:07:50,590 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:07:50,590 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:07:51,444 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:07:51,444 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:07:51,445 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:07:51,445 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:07:51,449 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:07:51,449 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:07:51,451 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:07:51,451 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:07:51,457 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:07:51,457 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:07:51,458 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:07:51,458 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:07:51,461 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:07:51,461 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:07:51,463 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:07:51,463 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:07:51,465 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:07:51,465 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:07:51,467 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:07:51,467 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:07:51,468 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:07:51,468 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:07:51,470 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:07:51,470 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:07:51,472 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:07:51,472 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:07:51,473 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:07:51,473 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:07:51,476 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:07:51,476 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:07:51,478 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:07:51,478 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:07:51,481 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:07:51,481 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:07:51,482 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:07:51,482 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:07:51,484 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:07:51,484 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:07:51,486 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:07:51,486 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:07:51,488 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:07:51,488 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:07:51,491 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:07:51,491 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:07:51,493 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:07:51,493 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:07:51,496 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:07:51,496 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:07:51,499 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:07:51,499 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:07:51,502 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:07:51,502 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:07:51,506 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:07:51,506 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:07:51,510 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:07:51,510 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:07:51,514 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:07:51,514 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:07:51,517 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:07:51,517 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:07:51,520 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:07:51,520 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:07:51,524 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:07:51,524 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:07:51,528 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:07:51,528 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:07:51,530 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:07:51,530 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:07:51,533 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:07:51,533 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:07:51,536 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:07:51,536 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:07:51,539 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:07:51,539 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:07:51,541 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:07:51,541 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:07:51,544 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:07:51,544 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:07:51,547 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:07:51,547 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:07:51,550 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:07:51,550 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:07:51,554 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:07:51,554 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:07:51,557 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:07:51,557 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:07:51,560 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:07:51,560 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:07:51,563 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:07:51,563 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:07:51,567 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:07:51,567 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:07:51,571 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:07:51,571 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:07:51,573 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:07:51,573 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:07:51,578 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:07:51,578 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:07:51,582 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:07:51,582 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:07:51,585 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:07:51,585 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:07:51,587 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:07:51,587 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:07:51,594 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:07:51,594 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:07:51,597 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:07:51,597 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:07:51,602 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:07:51,602 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:07:51,650 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:07:51,650 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:07:51,652 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:07:51,652 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:07:51,657 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:07:51,657 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:07:51,658 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:07:51,658 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:07:51,665 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:07:51,665 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:07:52,259 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:07:52,259 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:07:52,749 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:07:52,749 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:07:53,289 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:07:53,289 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:07:53,814 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:07:53,814 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:07:54,318 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:07:54,318 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:07:54,805 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:07:54,805 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:07:55,309 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:07:55,309 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:07:55,751 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:07:55,751 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:07:56,229 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:07:56,229 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:07:56,682 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:07:56,682 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:07:57,188 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:07:57,188 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:07:57,683 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:07:57,683 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:07:58,176 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:07:58,176 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:07:58,661 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:07:58,661 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:07:59,125 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:07:59,125 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:07:59,660 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:07:59,660 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:08:00,141 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:08:00,141 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:08:00,621 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:08:00,621 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:08:01,060 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:08:01,060 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:08:01,550 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:08:01,550 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:08:02,018 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:08:02,018 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:08:02,507 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:08:02,507 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:08:02,992 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:08:02,992 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:08:03,463 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:08:03,463 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:08:03,954 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:08:03,954 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:08:04,441 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:08:04,441 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:08:04,974 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:08:04,974 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:08:05,449 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:08:05,449 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:08:05,927 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:08:05,927 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:08:06,420 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:08:06,420 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:08:06,946 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:08:06,946 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:08:07,434 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:08:07,434 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:08:07,930 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:08:07,930 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:08:08,441 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:08:08,441 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:08:08,875 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:08:08,875 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:08:09,372 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:08:09,372 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:08:09,877 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:08:09,877 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:08:10,382 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:08:10,382 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:08:10,861 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:08:10,861 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:08:11,335 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:08:11,335 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:08:11,804 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:08:11,804 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:08:12,227 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:08:12,227 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:08:12,780 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:08:12,780 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:08:13,274 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:08:13,274 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:08:13,692 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:08:13,692 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:08:14,238 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:08:14,238 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:08:14,827 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:08:14,827 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:08:15,451 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:08:15,451 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:08:16,001 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:08:16,001 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:08:16,611 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:08:16,611 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:08:17,473 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:08:17,473 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:08:17,474 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:08:17,474 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:08:17,479 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:08:17,479 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:08:17,481 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:08:17,481 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:08:17,486 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:08:17,486 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:08:18,093 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:08:18,093 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:08:18,743 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:08:18,743 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:08:19,466 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:08:19,466 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:08:20,033 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:08:20,033 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:08:20,585 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:08:20,585 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:08:21,064 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:08:21,064 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:08:21,596 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:08:21,596 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:08:22,072 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:08:22,072 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:08:22,583 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:08:22,583 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:08:23,074 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:08:23,074 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:08:23,691 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:08:23,691 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:08:24,227 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:08:24,227 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:08:24,762 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:08:24,762 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:08:25,262 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:08:25,262 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:08:25,748 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:08:25,748 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:08:26,305 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:08:26,305 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:08:26,820 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:08:26,820 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:08:27,326 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:08:27,326 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:08:27,791 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:08:27,791 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:08:28,299 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:08:28,299 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:08:28,818 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:08:28,818 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:08:29,346 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:08:29,346 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:08:29,862 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:08:29,862 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:08:30,370 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:08:30,370 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:08:30,896 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:08:30,896 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:08:31,401 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:08:31,401 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:08:31,926 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:08:31,926 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:08:32,577 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:08:32,577 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:08:33,104 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:08:33,104 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:08:33,647 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:08:33,647 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:08:34,257 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:08:34,257 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:08:34,821 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:08:34,821 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:08:35,411 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:08:35,411 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:08:35,983 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:08:35,983 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:08:36,460 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:08:36,460 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:08:37,002 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:08:37,002 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:08:37,535 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:08:37,535 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:08:38,017 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:08:38,017 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:08:38,546 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:08:38,546 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:08:39,086 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:08:39,086 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:08:39,628 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:08:39,628 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:08:40,099 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:08:40,099 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:08:40,662 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:08:40,662 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:08:41,208 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:08:41,208 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:08:41,651 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:08:41,651 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:08:42,156 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:08:42,156 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:08:42,746 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:08:42,746 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:08:43,622 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:08:43,622 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:08:44,355 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:08:44,355 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:08:45,178 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:08:45,178 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:08:46,283 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:08:46,283 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:08:46,284 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:08:46,284 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:08:46,290 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:08:46,290 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:08:46,291 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:08:46,291 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:08:46,292 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:08:46,292 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:08:46,292 INFO Evaluation progress: 100% - DONE 2026-01-22 17:08:46,292 INFO Evaluation progress: 100% - DONE 2026-01-22 17:08:46,299 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom4/h2o-sonar/mli_experiment_277fabca-b4f5-49a1-8602-5a5638e9a000/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_7686cb59-df8d-47ee-816a-fc06b9f059cf/global_html_fragment/text_html/explanation.html' 2026-01-22 17:08:46,299 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom4/h2o-sonar/mli_experiment_277fabca-b4f5-49a1-8602-5a5638e9a000/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_7686cb59-df8d-47ee-816a-fc06b9f059cf/global_html_fragment/text_html/explanation.html' 2026-01-22 17:08:46,318 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom4/h2o-sonar/mli_experiment_277fabca-b4f5-49a1-8602-5a5638e9a000/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_7686cb59-df8d-47ee-816a-fc06b9f059cf/global_html_fragment/text_html/explanation.html' 2026-01-22 17:08:46,318 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom4/h2o-sonar/mli_experiment_277fabca-b4f5-49a1-8602-5a5638e9a000/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_7686cb59-df8d-47ee-816a-fc06b9f059cf/global_html_fragment/text_html/explanation.html' 2026-01-22 17:09:41,205 INFO Evaluation progress: 0% - Started 2026-01-22 17:09:41,205 INFO Evaluation progress: 0% - Started 2026-01-22 17:09:41,207 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:09:41,207 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:09:41,214 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 49, 1, 1, 50, 47, 47, 50, 50, 50, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:09:41,214 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 49, 1, 1, 50, 47, 47, 50, 50, 50, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:09:41,216 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:09:41,216 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:09:41,217 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:09:41,217 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:09:41,218 INFO COMPATIBLE 2026-01-22 17:09:41,218 INFO COMPATIBLE 2026-01-22 17:09:41,219 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:09:41,219 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:09:41,220 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:09:41,220 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:09:41,221 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:09:41,221 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:09:41,222 INFO COMPATIBLE 2026-01-22 17:09:41,222 INFO COMPATIBLE 2026-01-22 17:09:41,223 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:09:41,223 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:09:41,224 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:09:41,224 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:09:41,225 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:09:41,225 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:09:41,227 INFO COMPATIBLE 2026-01-22 17:09:41,227 INFO COMPATIBLE 2026-01-22 17:09:41,228 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:09:41,228 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:09:41,229 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:09:41,229 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:09:41,230 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:09:41,230 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:09:41,231 INFO COMPATIBLE 2026-01-22 17:09:41,231 INFO COMPATIBLE 2026-01-22 17:09:41,232 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:09:41,232 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:09:41,233 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:09:41,233 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:09:41,235 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:09:41,235 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:09:41,236 INFO COMPATIBLE 2026-01-22 17:09:41,236 INFO COMPATIBLE 2026-01-22 17:09:41,237 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:09:41,237 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:09:41,238 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:09:41,238 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:09:41,239 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:09:41,239 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:09:41,241 INFO COMPATIBLE 2026-01-22 17:09:41,241 INFO COMPATIBLE 2026-01-22 17:09:41,242 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:09:41,242 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:09:41,243 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:09:41,243 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:09:41,244 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:09:41,244 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:09:41,245 INFO COMPATIBLE 2026-01-22 17:09:41,245 INFO COMPATIBLE 2026-01-22 17:09:41,247 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:09:41,247 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:09:41,248 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:09:41,248 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:09:41,249 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:09:41,249 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:09:41,252 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:09:41,252 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:09:41,253 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:09:41,253 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:09:41,259 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:09:41,259 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:09:41,852 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:09:41,852 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:09:41,904 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:09:41,904 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:09:41,972 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:09:41,972 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:09:42,063 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:09:42,063 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:09:42,192 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:09:42,192 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:09:42,256 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:09:42,256 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:09:42,331 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:09:42,331 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:09:42,388 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:09:42,388 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:09:42,470 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:09:42,470 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:09:42,575 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:09:42,575 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:09:42,635 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:09:42,635 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:09:42,691 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:09:42,691 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:09:42,765 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:09:42,765 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:09:42,912 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:09:42,912 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:09:42,977 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:09:42,977 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:09:43,070 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:09:43,070 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:09:43,219 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:09:43,219 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:09:43,315 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:09:43,315 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:09:43,397 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:09:43,397 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:09:43,547 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:09:43,547 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:09:43,655 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:09:43,655 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:09:43,741 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:09:43,741 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:09:43,841 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:09:43,841 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:09:43,940 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:09:43,940 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:09:44,011 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:09:44,011 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:09:44,113 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:09:44,113 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:09:44,173 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:09:44,173 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:09:44,249 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:09:44,249 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:09:44,475 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:09:44,475 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:09:44,587 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:09:44,587 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:09:44,706 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:09:44,706 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:09:44,861 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:09:44,861 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:09:44,953 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:09:44,953 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:09:45,009 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:09:45,009 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:09:45,145 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:09:45,145 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:09:45,239 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:09:45,239 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:09:45,319 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:09:45,319 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:09:45,421 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:09:45,421 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:09:45,503 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:09:45,503 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:09:45,591 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:09:45,591 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:09:45,733 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:09:45,733 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:09:45,810 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:09:45,810 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:09:45,876 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:09:45,876 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:09:45,967 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:09:45,967 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:09:46,056 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:09:46,056 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:09:46,152 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:09:46,152 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:09:46,253 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:09:46,253 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:09:46,347 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:09:46,347 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:09:46,447 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:09:46,447 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:09:46,563 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:09:46,563 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:09:47,063 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:09:47,063 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:09:47,064 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:09:47,064 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:09:47,067 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:09:47,067 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:09:47,069 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:09:47,069 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:09:47,075 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:09:47,075 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:09:47,575 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:09:47,575 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:09:47,620 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:09:47,620 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:09:47,682 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:09:47,682 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:09:47,767 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:09:47,767 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:09:47,952 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:09:47,952 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:09:48,017 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:09:48,017 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:09:48,088 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:09:48,088 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:09:48,143 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:09:48,143 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:09:48,229 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:09:48,229 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:09:48,312 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:09:48,312 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:09:48,377 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:09:48,377 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:09:48,435 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:09:48,435 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:09:48,534 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:09:48,534 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:09:48,659 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:09:48,659 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:09:48,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:09:48,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:09:48,774 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:09:48,774 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:09:48,913 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:09:48,913 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:09:49,022 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:09:49,022 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:09:49,094 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:09:49,094 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:09:49,305 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:09:49,305 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:09:49,574 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:09:49,574 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:09:49,718 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:09:49,718 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:09:49,815 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:09:49,815 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:09:49,938 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:09:49,938 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:09:50,009 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:09:50,009 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:09:50,155 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:09:50,155 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:09:50,229 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:09:50,229 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:09:50,338 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:09:50,338 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:09:50,549 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:09:50,549 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:09:50,658 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:09:50,658 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:09:50,765 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:09:50,765 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:09:50,968 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:09:50,968 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:09:51,188 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:09:51,188 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:09:51,263 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:09:51,263 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:09:51,483 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:09:51,483 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:09:51,625 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:09:51,625 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:09:51,713 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:09:51,713 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:09:51,855 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:09:51,855 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:09:51,944 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:09:51,944 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:09:52,064 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:09:52,064 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:09:52,197 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:09:52,197 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:09:52,304 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:09:52,304 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:09:52,390 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:09:52,390 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:09:52,515 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:09:52,515 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:09:52,638 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:09:52,638 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:09:52,763 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:09:52,763 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:09:52,880 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:09:52,880 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:09:53,012 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:09:53,012 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:09:53,101 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:09:53,101 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:09:53,237 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:09:53,237 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:09:53,789 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:09:53,789 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:09:53,790 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:09:53,790 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:09:53,794 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:09:53,794 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:09:53,796 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:09:53,796 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:09:54,121 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:09:54,121 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:09:54,123 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:09:54,123 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:09:54,130 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:09:54,130 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:09:54,131 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:09:54,131 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:09:54,142 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:09:54,142 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:09:54,799 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:09:54,799 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:09:55,347 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:09:55,347 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:09:55,980 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:09:55,980 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:09:56,503 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:09:56,503 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:09:57,141 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:09:57,141 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:09:57,786 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:09:57,786 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:09:58,337 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:09:58,337 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:09:58,902 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:09:58,902 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:09:59,350 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:09:59,350 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:09:59,912 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:09:59,912 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:10:00,529 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:10:00,529 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:10:01,129 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:10:01,129 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:10:01,589 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:10:01,589 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:10:02,163 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:10:02,163 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:10:02,619 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:10:02,619 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:10:03,144 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:10:03,144 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:10:03,653 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:10:03,653 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:10:04,257 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:10:04,257 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:10:04,890 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:10:04,890 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:10:05,480 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:10:05,480 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:10:06,125 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:10:06,125 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:10:06,827 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:10:06,827 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:10:07,388 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:10:07,388 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:10:08,086 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:10:08,086 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:10:08,567 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:10:08,567 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:10:09,080 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:10:09,080 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:10:09,620 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:10:09,620 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:10:10,075 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:10:10,075 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:10:10,663 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:10:10,663 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:10:11,248 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:10:11,248 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:10:11,908 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:10:11,908 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:10:12,560 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:10:12,560 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:10:13,163 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:10:13,163 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:10:13,645 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:10:13,645 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:10:14,304 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:10:14,304 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:10:14,850 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:10:14,850 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:10:15,492 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:10:15,492 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:10:15,933 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:10:15,933 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:10:16,485 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:10:16,485 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:10:17,065 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:10:17,065 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:10:17,653 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:10:17,653 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:10:18,154 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:10:18,154 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:10:18,694 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:10:18,694 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:10:19,402 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:10:19,402 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:10:19,769 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:10:19,769 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:10:20,351 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:10:20,351 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:10:20,957 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:10:20,957 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:10:21,604 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:10:21,604 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:10:22,097 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:10:22,097 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:10:22,703 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:10:22,703 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:10:23,600 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:10:23,600 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:10:23,602 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:10:23,602 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:10:23,608 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:10:23,608 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:10:23,610 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:10:23,610 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:10:23,617 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:10:23,617 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:10:23,619 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:10:23,619 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:10:23,622 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:10:23,622 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:10:23,625 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:10:23,625 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:10:23,628 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:10:23,628 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:10:23,631 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:10:23,631 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:10:23,633 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:10:23,633 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:10:23,636 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:10:23,636 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:10:23,639 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:10:23,639 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:10:23,641 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:10:23,641 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:10:23,645 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:10:23,645 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:10:23,647 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:10:23,647 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:10:23,651 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:10:23,651 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:10:23,653 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:10:23,653 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:10:23,654 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:10:23,654 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:10:23,656 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:10:23,656 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:10:23,658 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:10:23,658 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:10:23,661 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:10:23,661 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:10:23,663 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:10:23,663 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:10:23,666 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:10:23,666 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:10:23,670 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:10:23,670 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:10:23,673 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:10:23,673 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:10:23,677 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:10:23,677 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:10:23,680 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:10:23,680 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:10:23,684 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:10:23,684 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:10:23,687 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:10:23,687 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:10:23,691 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:10:23,691 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:10:23,694 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:10:23,694 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:10:23,698 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:10:23,698 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:10:23,700 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:10:23,700 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:10:23,703 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:10:23,703 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:10:23,707 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:10:23,707 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:10:23,709 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:10:23,709 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:10:23,711 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:10:23,711 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:10:23,714 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:10:23,714 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:10:23,717 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:10:23,717 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:10:23,720 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:10:23,720 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:10:23,724 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:10:23,724 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:10:23,727 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:10:23,727 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:10:23,730 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:10:23,730 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:10:23,733 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:10:23,733 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:10:23,737 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:10:23,737 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:10:23,740 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:10:23,740 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:10:23,742 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:10:23,742 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:10:23,747 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:10:23,747 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:10:23,752 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:10:23,752 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:10:23,754 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:10:23,754 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:10:23,756 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:10:23,756 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:10:23,763 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:10:23,763 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:10:23,765 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:10:23,765 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:10:23,770 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:10:23,770 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:10:23,817 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:10:23,817 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:10:23,819 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:10:23,819 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:10:23,824 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:10:23,824 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:10:23,825 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:10:23,825 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:10:23,832 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:10:23,832 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:10:24,410 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:10:24,410 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:10:25,077 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:10:25,077 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:10:25,662 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:10:25,662 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:10:26,120 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:10:26,120 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:10:26,658 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:10:26,658 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:10:27,212 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:10:27,212 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:10:27,710 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:10:27,710 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:10:28,217 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:10:28,217 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:10:28,621 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:10:28,621 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:10:29,130 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:10:29,130 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:10:29,766 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:10:29,766 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:10:30,327 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:10:30,327 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:10:30,745 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:10:30,745 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:10:31,258 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:10:31,258 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:10:31,709 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:10:31,709 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:10:32,198 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:10:32,198 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:10:32,664 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:10:32,664 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:10:33,206 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:10:33,206 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:10:33,724 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:10:33,724 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:10:34,179 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:10:34,179 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:10:34,748 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:10:34,748 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:10:35,306 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:10:35,306 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:10:35,716 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:10:35,716 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:10:36,300 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:10:36,300 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:10:36,750 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:10:36,750 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:10:37,208 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:10:37,208 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:10:37,806 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:10:37,806 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:10:38,262 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:10:38,262 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:10:38,734 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:10:38,734 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:10:39,252 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:10:39,252 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:10:39,919 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:10:39,919 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:10:40,448 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:10:40,448 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:10:41,005 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:10:41,005 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:10:41,462 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:10:41,462 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:10:42,035 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:10:42,035 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:10:42,487 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:10:42,487 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:10:43,073 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:10:43,073 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:10:43,445 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:10:43,445 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:10:43,951 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:10:43,951 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:10:44,531 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:10:44,531 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:10:45,043 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:10:45,043 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:10:45,498 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:10:45,498 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:10:46,023 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:10:46,023 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:10:46,584 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:10:46,584 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:10:46,896 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:10:46,896 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:10:47,409 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:10:47,409 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:10:47,964 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:10:47,964 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:10:48,553 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:10:48,553 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:10:49,045 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:10:49,045 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:10:49,589 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:10:49,589 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:10:50,397 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:10:50,397 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:10:50,398 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:10:50,398 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:10:50,404 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:10:50,404 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:10:50,405 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:10:50,405 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:10:50,411 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:10:50,411 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:10:50,953 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:10:50,953 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:10:51,555 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:10:51,555 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:10:52,145 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:10:52,145 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:10:52,619 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:10:52,619 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:10:53,165 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:10:53,165 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:10:53,725 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:10:53,725 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:10:54,226 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:10:54,226 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:10:54,777 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:10:54,777 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:10:55,181 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:10:55,181 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:10:55,707 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:10:55,707 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:10:56,283 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:10:56,283 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:10:56,854 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:10:56,854 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:10:57,268 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:10:57,268 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:10:57,772 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:10:57,772 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:10:58,223 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:10:58,223 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:10:58,740 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:10:58,740 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:10:59,175 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:10:59,175 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:10:59,745 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:10:59,745 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:11:00,258 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:11:00,258 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:11:00,721 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:11:00,721 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:11:01,265 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:11:01,265 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:11:01,872 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:11:01,872 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:11:02,274 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:11:02,274 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:11:02,876 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:11:02,876 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:11:03,309 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:11:03,309 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:11:03,763 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:11:03,763 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:11:04,239 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:11:04,239 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:11:04,680 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:11:04,680 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:11:05,156 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:11:05,156 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:11:05,668 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:11:05,668 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:11:06,257 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:11:06,257 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:11:06,824 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:11:06,824 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:11:07,365 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:11:07,365 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:11:07,807 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:11:07,807 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:11:08,363 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:11:08,363 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:11:08,816 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:11:08,816 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:11:09,383 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:11:09,383 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:11:09,788 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:11:09,788 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:11:10,282 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:11:10,282 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:11:10,811 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:11:10,811 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:11:11,324 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:11:11,324 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:11:11,782 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:11:11,782 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:11:12,278 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:11:12,278 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:11:12,844 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:11:12,844 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:11:13,138 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:11:13,138 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:11:13,646 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:11:13,646 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:11:14,164 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:11:14,164 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:11:14,765 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:11:14,765 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:11:15,200 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:11:15,200 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:11:15,739 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:11:15,739 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:11:16,519 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:11:16,519 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:11:16,521 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:11:16,521 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:11:16,526 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:11:16,526 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:11:16,528 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:11:16,528 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:11:16,529 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:11:16,529 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:11:16,530 INFO Evaluation progress: 100% - DONE 2026-01-22 17:11:16,530 INFO Evaluation progress: 100% - DONE 2026-01-22 17:11:16,537 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_58eb5c72-1e1d-415a-a4ce-137fcef1670b/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_d37552e0-0f32-45aa-ab75-c352c5f1d34b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:11:16,537 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_58eb5c72-1e1d-415a-a4ce-137fcef1670b/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_d37552e0-0f32-45aa-ab75-c352c5f1d34b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:11:16,559 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_58eb5c72-1e1d-415a-a4ce-137fcef1670b/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_d37552e0-0f32-45aa-ab75-c352c5f1d34b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:11:16,559 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_58eb5c72-1e1d-415a-a4ce-137fcef1670b/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_d37552e0-0f32-45aa-ab75-c352c5f1d34b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:11:16,644 INFO Evaluation progress: 0% - Started 2026-01-22 17:11:16,644 INFO Evaluation progress: 0% - Started 2026-01-22 17:11:16,646 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:11:16,646 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:11:16,652 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 49, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:11:16,652 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 49, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:11:16,654 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:11:16,654 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:11:16,655 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:11:16,655 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:11:16,656 INFO COMPATIBLE 2026-01-22 17:11:16,656 INFO COMPATIBLE 2026-01-22 17:11:16,657 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:11:16,657 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:11:16,659 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:11:16,659 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:11:16,660 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:11:16,660 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:11:16,661 INFO COMPATIBLE 2026-01-22 17:11:16,661 INFO COMPATIBLE 2026-01-22 17:11:16,662 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:11:16,662 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:11:16,663 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:11:16,663 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:11:16,664 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:11:16,664 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:11:16,666 INFO COMPATIBLE 2026-01-22 17:11:16,666 INFO COMPATIBLE 2026-01-22 17:11:16,667 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:11:16,667 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:11:16,668 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:11:16,668 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:11:16,669 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:11:16,669 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:11:16,670 INFO COMPATIBLE 2026-01-22 17:11:16,670 INFO COMPATIBLE 2026-01-22 17:11:16,671 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:11:16,671 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:11:16,673 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:11:16,673 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:11:16,674 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:11:16,674 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:11:16,675 INFO COMPATIBLE 2026-01-22 17:11:16,675 INFO COMPATIBLE 2026-01-22 17:11:16,676 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:11:16,676 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:11:16,677 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:11:16,677 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:11:16,679 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:11:16,679 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:11:16,680 INFO COMPATIBLE 2026-01-22 17:11:16,680 INFO COMPATIBLE 2026-01-22 17:11:16,681 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:11:16,681 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:11:16,683 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:11:16,683 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:11:16,684 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:11:16,684 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:11:16,685 INFO COMPATIBLE 2026-01-22 17:11:16,685 INFO COMPATIBLE 2026-01-22 17:11:16,686 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:11:16,686 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:11:16,688 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:11:16,688 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:11:16,689 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:11:16,689 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:11:16,692 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,692 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,693 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,693 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,700 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:11:16,700 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:11:17,261 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:11:17,261 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:11:17,371 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:11:17,371 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:11:17,492 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:11:17,492 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:11:17,622 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:11:17,622 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:11:17,747 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:11:17,747 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:11:17,830 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:11:17,830 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:11:17,897 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:11:17,897 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:11:17,977 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:11:17,977 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:11:18,042 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:11:18,042 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:11:18,100 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:11:18,100 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:11:18,152 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:11:18,152 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:11:18,217 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:11:18,217 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:11:18,282 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:11:18,282 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:11:18,404 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:11:18,404 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:11:18,454 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:11:18,454 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:11:18,553 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:11:18,553 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:11:18,671 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:11:18,671 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:11:18,766 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:11:18,766 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:11:18,831 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:11:18,831 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:11:18,920 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:11:18,920 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:11:19,004 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:11:19,004 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:11:19,082 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:11:19,082 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:11:19,177 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:11:19,177 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:11:19,333 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:11:19,333 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:11:19,396 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:11:19,396 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:11:19,474 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:11:19,474 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:11:19,542 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:11:19,542 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:11:19,626 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:11:19,626 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:11:19,713 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:11:19,713 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:11:19,775 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:11:19,775 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:11:19,890 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:11:19,890 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:11:19,973 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:11:19,973 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:11:20,049 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:11:20,049 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:11:20,091 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:11:20,091 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:11:20,205 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:11:20,205 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:11:20,293 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:11:20,293 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:11:20,360 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:11:20,360 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:11:20,438 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:11:20,438 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:11:20,510 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:11:20,510 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:11:20,631 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:11:20,631 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:11:20,753 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:11:20,753 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:11:20,882 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:11:20,882 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:11:20,968 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:11:20,968 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:11:21,051 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:11:21,051 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:11:21,123 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:11:21,123 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:11:21,237 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:11:21,237 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:11:21,315 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:11:21,315 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:11:21,440 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:11:21,440 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:11:21,534 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:11:21,534 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:11:21,634 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:11:21,634 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:11:22,038 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:11:22,038 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:11:22,039 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:11:22,039 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:11:22,042 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,042 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,044 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,044 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,049 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:11:22,049 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:11:22,728 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:11:22,728 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:11:22,829 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:11:22,829 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:11:22,961 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:11:22,961 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:11:23,073 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:11:23,073 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:11:23,237 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:11:23,237 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:11:23,322 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:11:23,322 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:11:23,388 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:11:23,388 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:11:23,465 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:11:23,465 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:11:23,524 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:11:23,524 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:11:23,581 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:11:23,581 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:11:23,642 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:11:23,642 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:11:23,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:11:23,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:11:23,803 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:11:23,803 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:11:23,969 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:11:23,969 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:11:24,018 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:11:24,018 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:11:24,116 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:11:24,116 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:11:24,266 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:11:24,266 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:11:24,369 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:11:24,369 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:11:24,458 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:11:24,458 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:11:24,555 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:11:24,555 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:11:24,704 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:11:24,704 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:11:24,809 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:11:24,809 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:11:24,905 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:11:24,905 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:11:25,083 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:11:25,083 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:11:25,147 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:11:25,147 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:11:25,247 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:11:25,247 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:11:25,321 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:11:25,321 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:11:25,429 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:11:25,429 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:11:25,516 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:11:25,516 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:11:25,604 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:11:25,604 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:11:25,718 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:11:25,718 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:11:25,840 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:11:25,840 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:11:25,942 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:11:25,942 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:11:25,981 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:11:25,981 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:11:26,142 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:11:26,142 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:11:26,276 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:11:26,276 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:11:26,346 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:11:26,346 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:11:26,449 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:11:26,449 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:11:26,545 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:11:26,545 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:11:26,685 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:11:26,685 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:11:26,801 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:11:26,801 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:11:26,913 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:11:26,913 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:11:27,012 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:11:27,012 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:11:27,119 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:11:27,119 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:11:27,210 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:11:27,210 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:11:27,367 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:11:27,367 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:11:27,462 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:11:27,462 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:11:27,587 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:11:27,587 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:11:27,680 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:11:27,680 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:11:27,823 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:11:27,823 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:11:28,273 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:11:28,273 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:11:28,275 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:11:28,275 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:11:28,278 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,278 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,280 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,280 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,576 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:11:28,576 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:11:28,577 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:11:28,577 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:11:28,581 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,581 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,583 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,583 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,589 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:11:28,589 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:11:29,082 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:11:29,082 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:11:29,695 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:11:29,695 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:11:30,216 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:11:30,216 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:11:30,751 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:11:30,751 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:11:31,357 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:11:31,357 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:11:31,845 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:11:31,845 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:11:32,417 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:11:32,417 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:11:32,950 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:11:32,950 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:11:33,441 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:11:33,441 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:11:33,941 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:11:33,941 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:11:34,556 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:11:34,556 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:11:35,380 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:11:35,380 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:11:35,967 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:11:35,967 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:11:36,564 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:11:36,564 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:11:37,093 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:11:37,093 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:11:37,643 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:11:37,643 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:11:38,327 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:11:38,327 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:11:38,873 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:11:38,873 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:11:39,393 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:11:39,393 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:11:39,999 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:11:39,999 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:11:40,556 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:11:40,556 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:11:41,106 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:11:41,106 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:11:41,622 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:11:41,622 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:11:42,186 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:11:42,186 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:11:42,729 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:11:42,729 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:11:43,333 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:11:43,333 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:11:43,934 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:11:43,934 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:11:44,491 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:11:44,491 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:11:45,064 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:11:45,064 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:11:45,545 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:11:45,545 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:11:46,158 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:11:46,158 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:11:46,690 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:11:46,690 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:11:47,252 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:11:47,252 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:11:47,763 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:11:47,763 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:11:48,284 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:11:48,284 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:11:48,811 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:11:48,811 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:11:49,322 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:11:49,322 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:11:49,813 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:11:49,813 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:11:50,343 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:11:50,343 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:11:50,928 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:11:50,928 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:11:51,436 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:11:51,436 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:11:51,950 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:11:51,950 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:11:52,540 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:11:52,540 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:11:53,125 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:11:53,125 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:11:53,571 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:11:53,571 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:11:54,192 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:11:54,192 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:11:54,959 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:11:54,959 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:11:55,645 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:11:55,645 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:11:56,300 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:11:56,300 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:11:56,892 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:11:56,892 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:11:57,730 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:11:57,730 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:11:57,731 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:11:57,731 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:11:57,736 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,736 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,737 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,737 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,744 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:11:57,744 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:11:57,746 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:11:57,746 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:11:57,749 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:11:57,749 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:11:57,751 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:11:57,751 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:11:57,754 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:11:57,754 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:11:57,755 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:11:57,755 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:11:57,757 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:11:57,757 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:11:57,759 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:11:57,759 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:11:57,761 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:11:57,761 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:11:57,763 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:11:57,763 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:11:57,766 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:11:57,766 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:11:57,767 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:11:57,767 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:11:57,770 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:11:57,770 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:11:57,772 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:11:57,772 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:11:57,774 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:11:57,774 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:11:57,776 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:11:57,776 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:11:57,778 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:11:57,778 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:11:57,781 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:11:57,781 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:11:57,782 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:11:57,782 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:11:57,786 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:11:57,786 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:11:57,789 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:11:57,789 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:11:57,792 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:11:57,792 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:11:57,796 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:11:57,796 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:11:57,799 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:11:57,799 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:11:57,803 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:11:57,803 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:11:57,806 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:11:57,806 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:11:57,809 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:11:57,809 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:11:57,813 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:11:57,813 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:11:57,816 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:11:57,816 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:11:57,818 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:11:57,818 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:11:57,821 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:11:57,821 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:11:57,824 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:11:57,824 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:11:57,826 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:11:57,826 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:11:57,828 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:11:57,828 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:11:57,831 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:11:57,831 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:11:57,834 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:11:57,834 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:11:57,837 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:11:57,837 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:11:57,841 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:11:57,841 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:11:57,844 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:11:57,844 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:11:57,847 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:11:57,847 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:11:57,850 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:11:57,850 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:11:57,853 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:11:57,853 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:11:57,857 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:11:57,857 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:11:57,859 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:11:57,859 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:11:57,864 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:11:57,864 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:11:57,868 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:11:57,868 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:11:57,871 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:11:57,871 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:11:57,873 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:11:57,873 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:11:57,880 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:11:57,880 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:11:57,882 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:11:57,882 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:11:57,886 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:11:57,886 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:11:57,934 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:11:57,934 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:11:57,935 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:11:57,935 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:11:57,941 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,941 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,942 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,942 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,949 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:11:57,949 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:11:58,507 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:11:58,507 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:11:59,100 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:11:59,100 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:11:59,595 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:11:59,595 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:12:00,054 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:12:00,054 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:12:00,545 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:12:00,545 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:12:00,979 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:12:00,979 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:12:01,487 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:12:01,487 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:12:01,959 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:12:01,959 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:12:02,390 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:12:02,390 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:12:02,840 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:12:02,840 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:12:03,316 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:12:03,316 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:12:03,935 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:12:03,935 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:12:04,449 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:12:04,449 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:12:04,994 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:12:04,994 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:12:05,453 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:12:05,453 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:12:05,913 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:12:05,913 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:12:06,425 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:12:06,425 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:12:06,904 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:12:06,904 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:12:07,357 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:12:07,357 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:12:07,806 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:12:07,806 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:12:08,256 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:12:08,256 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:12:08,747 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:12:08,747 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:12:09,214 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:12:09,214 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:12:09,675 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:12:09,675 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:12:10,181 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:12:10,181 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:12:10,688 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:12:10,688 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:12:11,214 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:12:11,214 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:12:11,711 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:12:11,711 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:12:12,157 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:12:12,157 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:12:12,605 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:12:12,605 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:12:13,132 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:12:13,132 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:12:13,597 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:12:13,597 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:12:14,072 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:12:14,072 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:12:14,742 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:12:14,742 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:12:15,210 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:12:15,210 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:12:15,679 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:12:15,679 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:12:16,141 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:12:16,141 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:12:16,563 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:12:16,563 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:12:17,040 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:12:17,040 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:12:17,526 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:12:17,526 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:12:17,946 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:12:17,946 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:12:18,371 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:12:18,371 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:12:18,873 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:12:18,873 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:12:19,369 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:12:19,369 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:12:19,775 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:12:19,775 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:12:20,247 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:12:20,247 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:12:20,845 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:12:20,845 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:12:21,311 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:12:21,311 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:12:21,769 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:12:21,769 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:12:22,265 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:12:22,265 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:12:23,060 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:12:23,060 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:12:23,061 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:12:23,061 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:12:23,067 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,067 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,068 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,068 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,074 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:12:23,074 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:12:23,609 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:12:23,609 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:12:24,122 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:12:24,122 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:12:24,605 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:12:24,605 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:12:25,082 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:12:25,082 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:12:25,596 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:12:25,596 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:12:26,096 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:12:26,096 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:12:26,686 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:12:26,686 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:12:27,149 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:12:27,149 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:12:27,578 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:12:27,578 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:12:28,025 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:12:28,025 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:12:28,488 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:12:28,488 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:12:29,054 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:12:29,054 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:12:29,615 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:12:29,615 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:12:30,070 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:12:30,070 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:12:30,526 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:12:30,526 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:12:31,003 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:12:31,003 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:12:31,503 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:12:31,503 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:12:31,973 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:12:31,973 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:12:32,427 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:12:32,427 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:12:32,886 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:12:32,886 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:12:33,342 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:12:33,342 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:12:33,838 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:12:33,838 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:12:34,292 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:12:34,292 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:12:34,764 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:12:34,764 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:12:35,280 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:12:35,280 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:12:35,782 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:12:35,782 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:12:36,310 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:12:36,310 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:12:36,854 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:12:36,854 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:12:37,322 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:12:37,322 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:12:37,781 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:12:37,781 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:12:38,316 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:12:38,316 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:12:38,835 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:12:38,835 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:12:39,324 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:12:39,324 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:12:39,848 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:12:39,848 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:12:40,334 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:12:40,334 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:12:40,820 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:12:40,820 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:12:41,303 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:12:41,303 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:12:41,725 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:12:41,725 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:12:42,240 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:12:42,240 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:12:42,763 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:12:42,763 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:12:43,232 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:12:43,232 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:12:43,677 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:12:43,677 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:12:44,186 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:12:44,186 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:12:44,742 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:12:44,742 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:12:45,168 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:12:45,168 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:12:45,644 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:12:45,644 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:12:46,198 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:12:46,198 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:12:46,688 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:12:46,688 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:12:47,144 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:12:47,144 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:12:47,641 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:12:47,641 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:12:48,529 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:12:48,529 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:12:48,530 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:12:48,530 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:12:48,536 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:12:48,536 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:12:48,538 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:12:48,538 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:12:48,539 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:12:48,539 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:12:48,540 INFO Evaluation progress: 100% - DONE 2026-01-22 17:12:48,540 INFO Evaluation progress: 100% - DONE 2026-01-22 17:12:48,546 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:12:48,546 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:12:48,564 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:12:48,564 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html'