2026-01-22 17:09:41,205 INFO Evaluation progress: 0% - Started 2026-01-22 17:09:41,207 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:09:41,214 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 49, 1, 1, 50, 47, 47, 50, 50, 50, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:09:41,216 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:09:41,217 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:09:41,218 INFO COMPATIBLE 2026-01-22 17:09:41,219 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:09:41,220 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:09:41,221 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:09:41,222 INFO COMPATIBLE 2026-01-22 17:09:41,223 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:09:41,224 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:09:41,225 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:09:41,227 INFO COMPATIBLE 2026-01-22 17:09:41,228 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:09:41,229 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:09:41,230 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:09:41,231 INFO COMPATIBLE 2026-01-22 17:09:41,232 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:09:41,233 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:09:41,235 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:09:41,236 INFO COMPATIBLE 2026-01-22 17:09:41,237 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:09:41,238 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:09:41,239 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:09:41,241 INFO COMPATIBLE 2026-01-22 17:09:41,242 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:09:41,243 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:09:41,244 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:09:41,245 INFO COMPATIBLE 2026-01-22 17:09:41,247 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:09:41,248 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:09:41,249 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:09:41,252 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:09:41,253 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:09:41,259 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:09:41,852 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:09:41,904 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:09:41,972 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:09:42,063 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:09:42,192 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:09:42,256 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:09:42,331 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:09:42,388 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:09:42,470 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:09:42,575 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:09:42,635 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:09:42,691 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:09:42,765 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:09:42,912 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:09:42,977 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:09:43,070 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:09:43,219 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:09:43,315 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:09:43,397 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:09:43,547 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:09:43,655 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:09:43,741 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:09:43,841 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:09:43,940 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:09:44,011 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:09:44,113 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:09:44,173 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:09:44,249 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:09:44,475 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:09:44,587 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:09:44,706 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:09:44,861 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:09:44,953 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:09:45,009 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:09:45,145 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:09:45,239 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:09:45,319 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:09:45,421 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:09:45,503 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:09:45,591 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:09:45,733 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:09:45,810 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:09:45,876 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:09:45,967 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:09:46,056 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:09:46,152 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:09:46,253 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:09:46,347 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:09:46,447 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:09:46,563 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:09:47,063 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:09:47,064 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:09:47,067 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:09:47,069 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:09:47,075 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:09:47,575 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:09:47,620 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:09:47,682 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:09:47,767 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:09:47,952 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:09:48,017 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:09:48,088 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:09:48,143 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:09:48,229 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:09:48,312 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:09:48,377 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:09:48,435 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:09:48,534 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:09:48,659 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:09:48,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:09:48,774 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:09:48,913 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:09:49,022 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:09:49,094 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:09:49,305 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:09:49,574 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:09:49,718 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:09:49,815 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:09:49,938 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:09:50,009 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:09:50,155 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:09:50,229 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:09:50,338 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:09:50,549 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:09:50,658 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:09:50,765 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:09:50,968 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:09:51,188 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:09:51,263 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:09:51,483 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:09:51,625 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:09:51,713 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:09:51,855 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:09:51,944 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:09:52,064 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:09:52,197 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:09:52,304 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:09:52,390 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:09:52,515 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:09:52,638 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:09:52,763 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:09:52,880 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:09:53,012 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:09:53,101 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:09:53,237 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:09:53,789 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:09:53,790 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:09:53,794 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:09:53,796 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:09:54,121 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:09:54,123 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:09:54,130 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:09:54,131 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:09:54,142 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:09:54,799 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:09:55,347 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:09:55,980 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:09:56,503 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:09:57,141 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:09:57,786 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:09:58,337 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:09:58,902 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:09:59,350 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:09:59,912 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:10:00,529 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:10:01,129 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:10:01,589 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:10:02,163 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:10:02,619 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:10:03,144 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:10:03,653 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:10:04,257 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:10:04,890 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:10:05,480 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:10:06,125 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:10:06,827 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:10:07,388 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:10:08,086 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:10:08,567 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:10:09,080 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:10:09,620 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:10:10,075 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:10:10,663 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:10:11,248 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:10:11,908 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:10:12,560 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:10:13,163 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:10:13,645 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:10:14,304 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:10:14,850 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:10:15,492 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:10:15,933 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:10:16,485 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:10:17,065 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:10:17,653 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:10:18,154 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:10:18,694 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:10:19,402 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:10:19,769 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:10:20,351 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:10:20,957 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:10:21,604 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:10:22,097 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:10:22,703 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:10:23,600 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:10:23,602 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:10:23,608 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:10:23,610 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:10:23,617 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:10:23,619 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:10:23,622 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:10:23,625 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:10:23,628 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:10:23,631 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:10:23,633 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:10:23,636 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:10:23,639 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:10:23,641 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:10:23,645 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:10:23,647 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:10:23,651 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:10:23,653 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:10:23,654 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:10:23,656 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:10:23,658 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:10:23,661 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:10:23,663 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:10:23,666 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:10:23,670 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:10:23,673 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:10:23,677 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:10:23,680 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:10:23,684 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:10:23,687 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:10:23,691 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:10:23,694 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:10:23,698 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:10:23,700 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:10:23,703 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:10:23,707 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:10:23,709 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:10:23,711 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:10:23,714 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:10:23,717 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:10:23,720 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:10:23,724 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:10:23,727 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:10:23,730 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:10:23,733 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:10:23,737 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:10:23,740 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:10:23,742 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:10:23,747 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:10:23,752 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:10:23,754 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:10:23,756 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:10:23,763 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:10:23,765 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:10:23,770 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:10:23,817 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:10:23,819 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:10:23,824 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:10:23,825 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:10:23,832 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:10:24,410 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:10:25,077 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:10:25,662 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:10:26,120 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:10:26,658 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:10:27,212 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:10:27,710 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:10:28,217 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:10:28,621 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:10:29,130 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:10:29,766 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:10:30,327 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:10:30,745 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:10:31,258 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:10:31,709 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:10:32,198 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:10:32,664 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:10:33,206 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:10:33,724 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:10:34,179 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:10:34,748 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:10:35,306 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:10:35,716 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:10:36,300 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:10:36,750 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:10:37,208 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:10:37,806 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:10:38,262 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:10:38,734 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:10:39,252 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:10:39,919 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:10:40,448 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:10:41,005 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:10:41,462 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:10:42,035 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:10:42,487 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:10:43,073 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:10:43,445 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:10:43,951 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:10:44,531 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:10:45,043 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:10:45,498 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:10:46,023 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:10:46,584 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:10:46,896 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:10:47,409 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:10:47,964 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:10:48,553 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:10:49,045 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:10:49,589 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:10:50,397 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:10:50,398 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:10:50,404 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:10:50,405 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:10:50,411 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:10:50,953 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:10:51,555 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:10:52,145 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:10:52,619 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:10:53,165 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:10:53,725 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:10:54,226 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:10:54,777 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:10:55,181 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:10:55,707 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:10:56,283 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:10:56,854 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:10:57,268 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:10:57,772 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:10:58,223 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:10:58,740 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:10:59,175 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:10:59,745 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:11:00,258 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:11:00,721 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:11:01,265 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:11:01,872 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:11:02,274 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:11:02,876 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:11:03,309 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:11:03,763 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:11:04,239 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:11:04,680 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:11:05,156 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:11:05,668 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:11:06,257 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:11:06,824 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:11:07,365 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:11:07,807 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:11:08,363 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:11:08,816 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:11:09,383 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:11:09,788 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:11:10,282 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:11:10,811 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:11:11,324 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:11:11,782 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:11:12,278 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:11:12,844 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:11:13,138 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:11:13,646 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:11:14,164 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:11:14,765 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:11:15,200 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:11:15,739 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:11:16,519 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:11:16,521 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:11:16,526 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:11:16,528 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:11:16,529 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:11:16,530 INFO Evaluation progress: 100% - DONE 2026-01-22 17:11:16,537 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_58eb5c72-1e1d-415a-a4ce-137fcef1670b/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_d37552e0-0f32-45aa-ab75-c352c5f1d34b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:11:16,559 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_58eb5c72-1e1d-415a-a4ce-137fcef1670b/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_d37552e0-0f32-45aa-ab75-c352c5f1d34b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:11:16,644 INFO Evaluation progress: 0% - Started 2026-01-22 17:11:16,644 INFO Evaluation progress: 0% - Started 2026-01-22 17:11:16,646 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:11:16,646 INFO Evaluation progress: 1% - Preparation... 2026-01-22 17:11:16,652 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 49, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:11:16,652 INFO Running interpretation of explainable model: Model: None / Model handle: None Models: dict_values([]) Dataset: { "data": "", "metadata": { "shape": "(50, 14)", "row_count": 50, "column_names": [ "key", "input", "corpus", "context", "categories", "relationships", "expected_output", "output_constraints", "output_condition", "actual_output", "actual_duration", "cost", "model_key", "test_key" ], "column_types": [ "str", "str", "str", "str", "str", "str", "str", "str", "str", "str", "real", "real", "str", "str" ], "column_uniques": [ 50, 49, 1, 50, 1, 1, 50, 47, 47, 50, 50, 49, 1, 1 ], "columns_cat": [], "columns_num": [], "file_path": "", "file_name": "", "file_size": 0, "missing_values": [ "", "?", "None", "nan", "NA", "N/A", "unknown", "inf", "-inf", "1.7976931348623157e+308", "-1.7976931348623157e+308" ], "columns_meta": [ { "name": "key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "input", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "corpus", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "context", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "categories", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "relationships", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "expected_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_constraints", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "output_condition", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 47, "frequency": 0, "unique": 47, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_output", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "actual_duration", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 50, "frequency": 0, "unique": 50, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "cost", "data_type": "real", "logical_types": [], "format": "", "is_id": false, "is_numeric": true, "is_categorical": false, "count": 49, "frequency": 0, "unique": 49, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "model_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] }, { "name": "test_key", "data_type": "str", "logical_types": [], "format": "", "is_id": false, "is_numeric": false, "is_categorical": true, "count": 1, "frequency": 0, "unique": 1, "max": null, "min": null, "mean": null, "std": null, "histogram_counts": [], "histogram_ticks": [] } ], "original_dataset_sampled": false, "original_dataset_path": "", "original_dataset_size": 0, "original_dataset_shape": [ 50, 14 ] } } Dataset handle: None 2026-01-22 17:11:16,654 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:11:16,654 INFO Evaluation progress: 1% - #1/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator ... 2026-01-22 17:11:16,655 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:11:16,655 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator' ... 2026-01-22 17:11:16,656 INFO COMPATIBLE 2026-01-22 17:11:16,656 INFO COMPATIBLE 2026-01-22 17:11:16,657 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:11:16,657 INFO Evaluation progress: 2% - #1/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator finished 2026-01-22 17:11:16,659 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:11:16,659 INFO Evaluation progress: 2% - #2/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator ... 2026-01-22 17:11:16,660 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:11:16,660 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator' ... 2026-01-22 17:11:16,661 INFO COMPATIBLE 2026-01-22 17:11:16,661 INFO COMPATIBLE 2026-01-22 17:11:16,662 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:11:16,662 INFO Evaluation progress: 3% - #2/7 compatibility check of the evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator finished 2026-01-22 17:11:16,663 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:11:16,663 INFO Evaluation progress: 3% - #3/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator ... 2026-01-22 17:11:16,664 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:11:16,664 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator' ... 2026-01-22 17:11:16,666 INFO COMPATIBLE 2026-01-22 17:11:16,666 INFO COMPATIBLE 2026-01-22 17:11:16,667 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:11:16,667 INFO Evaluation progress: 4% - #3/7 compatibility check of the evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator finished 2026-01-22 17:11:16,668 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:11:16,668 INFO Evaluation progress: 4% - #4/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator ... 2026-01-22 17:11:16,669 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:11:16,669 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator' ... 2026-01-22 17:11:16,670 INFO COMPATIBLE 2026-01-22 17:11:16,670 INFO COMPATIBLE 2026-01-22 17:11:16,671 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:11:16,671 INFO Evaluation progress: 5% - #4/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator finished 2026-01-22 17:11:16,673 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:11:16,673 INFO Evaluation progress: 5% - #5/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator ... 2026-01-22 17:11:16,674 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:11:16,674 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator' ... 2026-01-22 17:11:16,675 INFO COMPATIBLE 2026-01-22 17:11:16,675 INFO COMPATIBLE 2026-01-22 17:11:16,676 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:11:16,676 INFO Evaluation progress: 6% - #5/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator finished 2026-01-22 17:11:16,677 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:11:16,677 INFO Evaluation progress: 6% - #6/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator ... 2026-01-22 17:11:16,679 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:11:16,679 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator' ... 2026-01-22 17:11:16,680 INFO COMPATIBLE 2026-01-22 17:11:16,680 INFO COMPATIBLE 2026-01-22 17:11:16,681 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:11:16,681 INFO Evaluation progress: 7% - #6/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator finished 2026-01-22 17:11:16,683 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:11:16,683 INFO Evaluation progress: 7% - #7/7 checking compatibility of the explainer/evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator ... 2026-01-22 17:11:16,684 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:11:16,684 INFO Checking compatibility of explainer: 'h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator' ... 2026-01-22 17:11:16,685 INFO COMPATIBLE 2026-01-22 17:11:16,685 INFO COMPATIBLE 2026-01-22 17:11:16,686 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:11:16,686 INFO Evaluation progress: 9% - #7/7 compatibility check of the evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator finished 2026-01-22 17:11:16,688 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:11:16,688 INFO Evaluation progress: 10% - Prepared 2026-01-22 17:11:16,689 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:11:16,689 INFO Execution plan tree: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator +-h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator +-h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator +-h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator +-h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator +-h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator `-h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:11:16,692 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,692 DEBUG Running explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,693 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,693 INFO Evaluation progress: 10% - 1/7 setting up evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator 2026-01-22 17:11:16,700 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:11:16,700 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Configuring metrics... 2026-01-22 17:11:17,261 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:11:17,261 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 1/50 2026-01-22 17:11:17,371 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:11:17,371 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 2/50 2026-01-22 17:11:17,492 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:11:17,492 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 3/50 2026-01-22 17:11:17,622 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:11:17,622 INFO Evaluation progress: 10% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 4/50 2026-01-22 17:11:17,747 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:11:17,747 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 5/50 2026-01-22 17:11:17,830 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:11:17,830 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 6/50 2026-01-22 17:11:17,897 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:11:17,897 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 7/50 2026-01-22 17:11:17,977 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:11:17,977 INFO Evaluation progress: 11% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 8/50 2026-01-22 17:11:18,042 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:11:18,042 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 9/50 2026-01-22 17:11:18,100 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:11:18,100 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 10/50 2026-01-22 17:11:18,152 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:11:18,152 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 11/50 2026-01-22 17:11:18,217 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:11:18,217 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 12/50 2026-01-22 17:11:18,282 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:11:18,282 INFO Evaluation progress: 12% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 13/50 2026-01-22 17:11:18,404 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:11:18,404 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 14/50 2026-01-22 17:11:18,454 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:11:18,454 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 15/50 2026-01-22 17:11:18,553 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:11:18,553 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 16/50 2026-01-22 17:11:18,671 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:11:18,671 INFO Evaluation progress: 13% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 17/50 2026-01-22 17:11:18,766 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:11:18,766 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 18/50 2026-01-22 17:11:18,831 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:11:18,831 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 19/50 2026-01-22 17:11:18,920 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:11:18,920 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 20/50 2026-01-22 17:11:19,004 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:11:19,004 INFO Evaluation progress: 14% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 21/50 2026-01-22 17:11:19,082 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:11:19,082 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 22/50 2026-01-22 17:11:19,177 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:11:19,177 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 23/50 2026-01-22 17:11:19,333 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:11:19,333 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 24/50 2026-01-22 17:11:19,396 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:11:19,396 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 25/50 2026-01-22 17:11:19,474 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:11:19,474 INFO Evaluation progress: 15% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 26/50 2026-01-22 17:11:19,542 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:11:19,542 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 27/50 2026-01-22 17:11:19,626 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:11:19,626 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 28/50 2026-01-22 17:11:19,713 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:11:19,713 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 29/50 2026-01-22 17:11:19,775 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:11:19,775 INFO Evaluation progress: 16% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 30/50 2026-01-22 17:11:19,890 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:11:19,890 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 31/50 2026-01-22 17:11:19,973 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:11:19,973 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 32/50 2026-01-22 17:11:20,049 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:11:20,049 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 33/50 2026-01-22 17:11:20,091 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:11:20,091 INFO Evaluation progress: 17% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 34/50 2026-01-22 17:11:20,205 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:11:20,205 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 35/50 2026-01-22 17:11:20,293 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:11:20,293 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 36/50 2026-01-22 17:11:20,360 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:11:20,360 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 37/50 2026-01-22 17:11:20,438 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:11:20,438 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 38/50 2026-01-22 17:11:20,510 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:11:20,510 INFO Evaluation progress: 18% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 39/50 2026-01-22 17:11:20,631 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:11:20,631 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 40/50 2026-01-22 17:11:20,753 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:11:20,753 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 41/50 2026-01-22 17:11:20,882 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:11:20,882 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 42/50 2026-01-22 17:11:20,968 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:11:20,968 INFO Evaluation progress: 19% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 43/50 2026-01-22 17:11:21,051 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:11:21,051 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 44/50 2026-01-22 17:11:21,123 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:11:21,123 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 45/50 2026-01-22 17:11:21,237 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:11:21,237 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 46/50 2026-01-22 17:11:21,315 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:11:21,315 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 47/50 2026-01-22 17:11:21,440 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:11:21,440 INFO Evaluation progress: 20% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 48/50 2026-01-22 17:11:21,534 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:11:21,534 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 49/50 2026-01-22 17:11:21,634 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:11:21,634 INFO Evaluation progress: 21% - Answer accuracy (semantic similarity) - Build > config > run 'answer_accuracy' (cpu) evaluation for input 50/50 2026-01-22 17:11:22,038 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:11:22,038 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator - created explanations: 3 2026-01-22 17:11:22,039 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:11:22,039 INFO Evaluation progress: 21% - 1/7 evaluator h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator run FINISHED 2026-01-22 17:11:22,042 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,042 DEBUG Running explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,044 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,044 INFO Evaluation progress: 21% - 2/7 setting up evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator 2026-01-22 17:11:22,049 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:11:22,049 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Configuring metrics... 2026-01-22 17:11:22,728 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:11:22,728 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 1/50 2026-01-22 17:11:22,829 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:11:22,829 INFO Evaluation progress: 21% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 2/50 2026-01-22 17:11:22,961 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:11:22,961 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 3/50 2026-01-22 17:11:23,073 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:11:23,073 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 4/50 2026-01-22 17:11:23,237 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:11:23,237 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 5/50 2026-01-22 17:11:23,322 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:11:23,322 INFO Evaluation progress: 22% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 6/50 2026-01-22 17:11:23,388 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:11:23,388 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 7/50 2026-01-22 17:11:23,465 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:11:23,465 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 8/50 2026-01-22 17:11:23,524 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:11:23,524 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 9/50 2026-01-22 17:11:23,581 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:11:23,581 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 10/50 2026-01-22 17:11:23,642 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:11:23,642 INFO Evaluation progress: 23% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 11/50 2026-01-22 17:11:23,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:11:23,705 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 12/50 2026-01-22 17:11:23,803 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:11:23,803 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 13/50 2026-01-22 17:11:23,969 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:11:23,969 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 14/50 2026-01-22 17:11:24,018 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:11:24,018 INFO Evaluation progress: 24% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 15/50 2026-01-22 17:11:24,116 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:11:24,116 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 16/50 2026-01-22 17:11:24,266 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:11:24,266 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 17/50 2026-01-22 17:11:24,369 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:11:24,369 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 18/50 2026-01-22 17:11:24,458 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:11:24,458 INFO Evaluation progress: 25% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 19/50 2026-01-22 17:11:24,555 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:11:24,555 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 20/50 2026-01-22 17:11:24,704 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:11:24,704 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 21/50 2026-01-22 17:11:24,809 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:11:24,809 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 22/50 2026-01-22 17:11:24,905 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:11:24,905 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 23/50 2026-01-22 17:11:25,083 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:11:25,083 INFO Evaluation progress: 26% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 24/50 2026-01-22 17:11:25,147 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:11:25,147 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 25/50 2026-01-22 17:11:25,247 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:11:25,247 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 26/50 2026-01-22 17:11:25,321 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:11:25,321 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 27/50 2026-01-22 17:11:25,429 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:11:25,429 INFO Evaluation progress: 27% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 28/50 2026-01-22 17:11:25,516 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:11:25,516 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 29/50 2026-01-22 17:11:25,604 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:11:25,604 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 30/50 2026-01-22 17:11:25,718 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:11:25,718 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 31/50 2026-01-22 17:11:25,840 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:11:25,840 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 32/50 2026-01-22 17:11:25,942 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:11:25,942 INFO Evaluation progress: 28% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 33/50 2026-01-22 17:11:25,981 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:11:25,981 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 34/50 2026-01-22 17:11:26,142 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:11:26,142 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 35/50 2026-01-22 17:11:26,276 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:11:26,276 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 36/50 2026-01-22 17:11:26,346 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:11:26,346 INFO Evaluation progress: 29% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 37/50 2026-01-22 17:11:26,449 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:11:26,449 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 38/50 2026-01-22 17:11:26,545 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:11:26,545 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 39/50 2026-01-22 17:11:26,685 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:11:26,685 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 40/50 2026-01-22 17:11:26,801 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:11:26,801 INFO Evaluation progress: 30% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 41/50 2026-01-22 17:11:26,913 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:11:26,913 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 42/50 2026-01-22 17:11:27,012 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:11:27,012 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 43/50 2026-01-22 17:11:27,119 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:11:27,119 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 44/50 2026-01-22 17:11:27,210 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:11:27,210 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 45/50 2026-01-22 17:11:27,367 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:11:27,367 INFO Evaluation progress: 31% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 46/50 2026-01-22 17:11:27,462 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:11:27,462 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 47/50 2026-01-22 17:11:27,587 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:11:27,587 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 48/50 2026-01-22 17:11:27,680 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:11:27,680 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 49/50 2026-01-22 17:11:27,823 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:11:27,823 INFO Evaluation progress: 32% - Answer semantic sentence similarity - Build > config > run 'mean_answer_similarity' (cpu) evaluation for input 50/50 2026-01-22 17:11:28,273 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:11:28,273 DEBUG DONE execution of explainer: h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator - created explanations: 4 2026-01-22 17:11:28,275 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:11:28,275 INFO Evaluation progress: 32% - 2/7 evaluator h2o_sonar.evaluators.answer_semantic_similarity_per_sentence_evaluator.AnswerSemanticSimilarityPerSentenceEvaluator run FINISHED 2026-01-22 17:11:28,278 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,278 DEBUG Running explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,280 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,280 INFO Evaluation progress: 32% - 3/7 setting up evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator 2026-01-22 17:11:28,576 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:11:28,576 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator - created explanations: 3 2026-01-22 17:11:28,577 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:11:28,577 INFO Evaluation progress: 44% - 3/7 evaluator h2o_sonar.evaluators.rouge_evaluator.RougeEvaluator run FINISHED 2026-01-22 17:11:28,581 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,581 DEBUG Running explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,583 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,583 INFO Evaluation progress: 44% - 4/7 setting up evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator 2026-01-22 17:11:28,589 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:11:28,589 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Configuring metrics... 2026-01-22 17:11:29,082 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:11:29,082 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 1/50 2026-01-22 17:11:29,695 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:11:29,695 INFO Evaluation progress: 44% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 2/50 2026-01-22 17:11:30,216 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:11:30,216 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 3/50 2026-01-22 17:11:30,751 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:11:30,751 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 4/50 2026-01-22 17:11:31,357 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:11:31,357 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 5/50 2026-01-22 17:11:31,845 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:11:31,845 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 6/50 2026-01-22 17:11:32,417 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:11:32,417 INFO Evaluation progress: 45% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 7/50 2026-01-22 17:11:32,950 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:11:32,950 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 8/50 2026-01-22 17:11:33,441 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:11:33,441 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 9/50 2026-01-22 17:11:33,941 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:11:33,941 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 10/50 2026-01-22 17:11:34,556 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:11:34,556 INFO Evaluation progress: 46% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 11/50 2026-01-22 17:11:35,380 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:11:35,380 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 12/50 2026-01-22 17:11:35,967 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:11:35,967 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 13/50 2026-01-22 17:11:36,564 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:11:36,564 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 14/50 2026-01-22 17:11:37,093 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:11:37,093 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 15/50 2026-01-22 17:11:37,643 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:11:37,643 INFO Evaluation progress: 47% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 16/50 2026-01-22 17:11:38,327 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:11:38,327 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 17/50 2026-01-22 17:11:38,873 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:11:38,873 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 18/50 2026-01-22 17:11:39,393 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:11:39,393 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 19/50 2026-01-22 17:11:39,999 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:11:39,999 INFO Evaluation progress: 48% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 20/50 2026-01-22 17:11:40,556 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:11:40,556 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 21/50 2026-01-22 17:11:41,106 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:11:41,106 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 22/50 2026-01-22 17:11:41,622 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:11:41,622 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 23/50 2026-01-22 17:11:42,186 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:11:42,186 INFO Evaluation progress: 49% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 24/50 2026-01-22 17:11:42,729 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:11:42,729 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 25/50 2026-01-22 17:11:43,333 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:11:43,333 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 26/50 2026-01-22 17:11:43,934 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:11:43,934 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 27/50 2026-01-22 17:11:44,491 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:11:44,491 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 28/50 2026-01-22 17:11:45,064 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:11:45,064 INFO Evaluation progress: 50% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 29/50 2026-01-22 17:11:45,545 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:11:45,545 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 30/50 2026-01-22 17:11:46,158 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:11:46,158 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 31/50 2026-01-22 17:11:46,690 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:11:46,690 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 32/50 2026-01-22 17:11:47,252 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:11:47,252 INFO Evaluation progress: 51% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 33/50 2026-01-22 17:11:47,763 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:11:47,763 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 34/50 2026-01-22 17:11:48,284 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:11:48,284 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 35/50 2026-01-22 17:11:48,811 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:11:48,811 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 36/50 2026-01-22 17:11:49,322 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:11:49,322 INFO Evaluation progress: 52% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 37/50 2026-01-22 17:11:49,813 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:11:49,813 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 38/50 2026-01-22 17:11:50,343 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:11:50,343 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 39/50 2026-01-22 17:11:50,928 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:11:50,928 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 40/50 2026-01-22 17:11:51,436 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:11:51,436 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 41/50 2026-01-22 17:11:51,950 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:11:51,950 INFO Evaluation progress: 53% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 42/50 2026-01-22 17:11:52,540 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:11:52,540 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 43/50 2026-01-22 17:11:53,125 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:11:53,125 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 44/50 2026-01-22 17:11:53,571 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:11:53,571 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 45/50 2026-01-22 17:11:54,192 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:11:54,192 INFO Evaluation progress: 54% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 46/50 2026-01-22 17:11:54,959 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:11:54,959 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 47/50 2026-01-22 17:11:55,645 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:11:55,645 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 48/50 2026-01-22 17:11:56,300 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:11:56,300 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 49/50 2026-01-22 17:11:56,892 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:11:56,892 INFO Evaluation progress: 55% - Groundedness (semantic similarity) - Build > config > run 'groundedness' (cpu) evaluation for input 50/50 2026-01-22 17:11:57,730 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:11:57,730 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator - created explanations: 4 2026-01-22 17:11:57,731 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:11:57,731 INFO Evaluation progress: 55% - 4/7 evaluator h2o_sonar.evaluators.rag_groundedness_evaluator.RagGroundednessEvaluator run FINISHED 2026-01-22 17:11:57,736 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,736 DEBUG Running explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,737 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,737 INFO Evaluation progress: 55% - 5/7 setting up evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator 2026-01-22 17:11:57,744 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:11:57,744 INFO Evaluation progress: 55% - Text matching - Checking conditions for 50 test cases 2026-01-22 17:11:57,746 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:11:57,746 INFO Evaluation progress: 56% - Text matching - Checked 1/50 test cases conditions 2026-01-22 17:11:57,749 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:11:57,749 INFO Evaluation progress: 56% - Text matching - Checked 2/50 test cases conditions 2026-01-22 17:11:57,751 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:11:57,751 INFO Evaluation progress: 56% - Text matching - Checked 3/50 test cases conditions 2026-01-22 17:11:57,754 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:11:57,754 INFO Evaluation progress: 56% - Text matching - Checked 4/50 test cases conditions 2026-01-22 17:11:57,755 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:11:57,755 INFO Evaluation progress: 56% - Text matching - Checked 5/50 test cases conditions 2026-01-22 17:11:57,757 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:11:57,757 INFO Evaluation progress: 57% - Text matching - Checked 6/50 test cases conditions 2026-01-22 17:11:57,759 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:11:57,759 INFO Evaluation progress: 57% - Text matching - Checked 7/50 test cases conditions 2026-01-22 17:11:57,761 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:11:57,761 INFO Evaluation progress: 57% - Text matching - Checked 8/50 test cases conditions 2026-01-22 17:11:57,763 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:11:57,763 INFO Evaluation progress: 57% - Text matching - Checked 9/50 test cases conditions 2026-01-22 17:11:57,766 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:11:57,766 INFO Evaluation progress: 58% - Text matching - Checked 10/50 test cases conditions 2026-01-22 17:11:57,767 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:11:57,767 INFO Evaluation progress: 58% - Text matching - Checked 11/50 test cases conditions 2026-01-22 17:11:57,770 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:11:57,770 INFO Evaluation progress: 58% - Text matching - Checked 12/50 test cases conditions 2026-01-22 17:11:57,772 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:11:57,772 INFO Evaluation progress: 58% - Text matching - Checked 13/50 test cases conditions 2026-01-22 17:11:57,774 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:11:57,774 INFO Evaluation progress: 58% - Text matching - Checked 14/50 test cases conditions 2026-01-22 17:11:57,776 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:11:57,776 INFO Evaluation progress: 59% - Text matching - Checked 15/50 test cases conditions 2026-01-22 17:11:57,778 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:11:57,778 INFO Evaluation progress: 59% - Text matching - Checked 16/50 test cases conditions 2026-01-22 17:11:57,781 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:11:57,781 INFO Evaluation progress: 59% - Text matching - Checked 17/50 test cases conditions 2026-01-22 17:11:57,782 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:11:57,782 INFO Evaluation progress: 59% - Text matching - Checked 18/50 test cases conditions 2026-01-22 17:11:57,786 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:11:57,786 INFO Evaluation progress: 60% - Text matching - Checked 19/50 test cases conditions 2026-01-22 17:11:57,789 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:11:57,789 INFO Evaluation progress: 60% - Text matching - Checked 20/50 test cases conditions 2026-01-22 17:11:57,792 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:11:57,792 INFO Evaluation progress: 60% - Text matching - Checked 21/50 test cases conditions 2026-01-22 17:11:57,796 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:11:57,796 INFO Evaluation progress: 60% - Text matching - Checked 22/50 test cases conditions 2026-01-22 17:11:57,799 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:11:57,799 INFO Evaluation progress: 61% - Text matching - Checked 23/50 test cases conditions 2026-01-22 17:11:57,803 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:11:57,803 INFO Evaluation progress: 61% - Text matching - Checked 24/50 test cases conditions 2026-01-22 17:11:57,806 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:11:57,806 INFO Evaluation progress: 61% - Text matching - Checked 25/50 test cases conditions 2026-01-22 17:11:57,809 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:11:57,809 INFO Evaluation progress: 61% - Text matching - Checked 26/50 test cases conditions 2026-01-22 17:11:57,813 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:11:57,813 INFO Evaluation progress: 61% - Text matching - Checked 27/50 test cases conditions 2026-01-22 17:11:57,816 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:11:57,816 INFO Evaluation progress: 62% - Text matching - Checked 28/50 test cases conditions 2026-01-22 17:11:57,818 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:11:57,818 INFO Evaluation progress: 62% - Text matching - Checked 29/50 test cases conditions 2026-01-22 17:11:57,821 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:11:57,821 INFO Evaluation progress: 62% - Text matching - Checked 30/50 test cases conditions 2026-01-22 17:11:57,824 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:11:57,824 INFO Evaluation progress: 62% - Text matching - Checked 31/50 test cases conditions 2026-01-22 17:11:57,826 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:11:57,826 INFO Evaluation progress: 63% - Text matching - Checked 32/50 test cases conditions 2026-01-22 17:11:57,828 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:11:57,828 INFO Evaluation progress: 63% - Text matching - Checked 33/50 test cases conditions 2026-01-22 17:11:57,831 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:11:57,831 INFO Evaluation progress: 63% - Text matching - Checked 34/50 test cases conditions 2026-01-22 17:11:57,834 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:11:57,834 INFO Evaluation progress: 63% - Text matching - Checked 35/50 test cases conditions 2026-01-22 17:11:57,837 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:11:57,837 INFO Evaluation progress: 63% - Text matching - Checked 36/50 test cases conditions 2026-01-22 17:11:57,841 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:11:57,841 INFO Evaluation progress: 64% - Text matching - Checked 37/50 test cases conditions 2026-01-22 17:11:57,844 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:11:57,844 INFO Evaluation progress: 64% - Text matching - Checked 38/50 test cases conditions 2026-01-22 17:11:57,847 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:11:57,847 INFO Evaluation progress: 64% - Text matching - Checked 39/50 test cases conditions 2026-01-22 17:11:57,850 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:11:57,850 INFO Evaluation progress: 64% - Text matching - Checked 40/50 test cases conditions 2026-01-22 17:11:57,853 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:11:57,853 INFO Evaluation progress: 65% - Text matching - Checked 41/50 test cases conditions 2026-01-22 17:11:57,857 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:11:57,857 INFO Evaluation progress: 65% - Text matching - Checked 42/50 test cases conditions 2026-01-22 17:11:57,859 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:11:57,859 INFO Evaluation progress: 65% - Text matching - Checked 43/50 test cases conditions 2026-01-22 17:11:57,864 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:11:57,864 INFO Evaluation progress: 65% - Text matching - Checked 44/50 test cases conditions 2026-01-22 17:11:57,868 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:11:57,868 INFO Evaluation progress: 66% - Text matching - Checked 45/50 test cases conditions 2026-01-22 17:11:57,871 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:11:57,871 INFO Evaluation progress: 66% - Text matching - Checked 46/50 test cases conditions 2026-01-22 17:11:57,873 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:11:57,873 INFO Evaluation progress: 66% - Text matching - Checked 47/50 test cases conditions 2026-01-22 17:11:57,880 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:11:57,880 INFO Evaluation progress: 66% - Text matching - Checked 48/50 test cases conditions 2026-01-22 17:11:57,882 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:11:57,882 INFO Evaluation progress: 66% - Text matching - Checked 49/50 test cases conditions 2026-01-22 17:11:57,886 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:11:57,886 INFO Evaluation progress: 67% - Text matching - Checked 50/50 test cases conditions 2026-01-22 17:11:57,934 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:11:57,934 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator - created explanations: 4 2026-01-22 17:11:57,935 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:11:57,935 INFO Evaluation progress: 67% - 5/7 evaluator h2o_sonar.evaluators.rag_tokens_presence_evaluator.RagStrStrEvaluator run FINISHED 2026-01-22 17:11:57,941 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,941 DEBUG Running explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,942 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,942 INFO Evaluation progress: 67% - 6/7 setting up evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator 2026-01-22 17:11:57,949 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:11:57,949 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Configuring metrics... 2026-01-22 17:11:58,507 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:11:58,507 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 1/50 2026-01-22 17:11:59,100 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:11:59,100 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 2/50 2026-01-22 17:11:59,595 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:11:59,595 INFO Evaluation progress: 67% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 3/50 2026-01-22 17:12:00,054 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:12:00,054 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 4/50 2026-01-22 17:12:00,545 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:12:00,545 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 5/50 2026-01-22 17:12:00,979 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:12:00,979 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 6/50 2026-01-22 17:12:01,487 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:12:01,487 INFO Evaluation progress: 68% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 7/50 2026-01-22 17:12:01,959 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:12:01,959 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 8/50 2026-01-22 17:12:02,390 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:12:02,390 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 9/50 2026-01-22 17:12:02,840 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:12:02,840 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 10/50 2026-01-22 17:12:03,316 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:12:03,316 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 11/50 2026-01-22 17:12:03,935 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:12:03,935 INFO Evaluation progress: 69% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 12/50 2026-01-22 17:12:04,449 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:12:04,449 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 13/50 2026-01-22 17:12:04,994 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:12:04,994 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 14/50 2026-01-22 17:12:05,453 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:12:05,453 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 15/50 2026-01-22 17:12:05,913 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:12:05,913 INFO Evaluation progress: 70% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 16/50 2026-01-22 17:12:06,425 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:12:06,425 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 17/50 2026-01-22 17:12:06,904 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:12:06,904 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 18/50 2026-01-22 17:12:07,357 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:12:07,357 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 19/50 2026-01-22 17:12:07,806 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:12:07,806 INFO Evaluation progress: 71% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 20/50 2026-01-22 17:12:08,256 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:12:08,256 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 21/50 2026-01-22 17:12:08,747 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:12:08,747 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 22/50 2026-01-22 17:12:09,214 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:12:09,214 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 23/50 2026-01-22 17:12:09,675 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:12:09,675 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 24/50 2026-01-22 17:12:10,181 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:12:10,181 INFO Evaluation progress: 72% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 25/50 2026-01-22 17:12:10,688 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:12:10,688 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 26/50 2026-01-22 17:12:11,214 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:12:11,214 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 27/50 2026-01-22 17:12:11,711 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:12:11,711 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 28/50 2026-01-22 17:12:12,157 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:12:12,157 INFO Evaluation progress: 73% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 29/50 2026-01-22 17:12:12,605 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:12:12,605 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 30/50 2026-01-22 17:12:13,132 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:12:13,132 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 31/50 2026-01-22 17:12:13,597 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:12:13,597 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 32/50 2026-01-22 17:12:14,072 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:12:14,072 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 33/50 2026-01-22 17:12:14,742 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:12:14,742 INFO Evaluation progress: 74% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 34/50 2026-01-22 17:12:15,210 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:12:15,210 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 35/50 2026-01-22 17:12:15,679 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:12:15,679 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 36/50 2026-01-22 17:12:16,141 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:12:16,141 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 37/50 2026-01-22 17:12:16,563 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:12:16,563 INFO Evaluation progress: 75% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 38/50 2026-01-22 17:12:17,040 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:12:17,040 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 39/50 2026-01-22 17:12:17,526 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:12:17,526 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 40/50 2026-01-22 17:12:17,946 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:12:17,946 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 41/50 2026-01-22 17:12:18,371 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:12:18,371 INFO Evaluation progress: 76% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 42/50 2026-01-22 17:12:18,873 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:12:18,873 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 43/50 2026-01-22 17:12:19,369 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:12:19,369 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 44/50 2026-01-22 17:12:19,775 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:12:19,775 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 45/50 2026-01-22 17:12:20,247 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:12:20,247 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 46/50 2026-01-22 17:12:20,845 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:12:20,845 INFO Evaluation progress: 77% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 47/50 2026-01-22 17:12:21,311 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:12:21,311 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 48/50 2026-01-22 17:12:21,769 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:12:21,769 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 49/50 2026-01-22 17:12:22,265 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:12:22,265 INFO Evaluation progress: 78% - Context relevancy (soft recall and precision) - Build > config > run 'recall_relevancy and precision_relevancy' (cpu) evaluation for input 50/50 2026-01-22 17:12:23,060 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:12:23,060 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator - created explanations: 4 2026-01-22 17:12:23,061 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:12:23,061 INFO Evaluation progress: 78% - 6/7 evaluator h2o_sonar.evaluators.rag_chunk_relevancy_evaluator.ContextChunkRelevancyEvaluator run FINISHED 2026-01-22 17:12:23,067 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,067 DEBUG Running explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,068 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,068 INFO Evaluation progress: 78% - 7/7 setting up evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator 2026-01-22 17:12:23,074 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:12:23,074 INFO Evaluation progress: 78% - Context mean reciprocal rank - Configuring metrics... 2026-01-22 17:12:23,609 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:12:23,609 INFO Evaluation progress: 78% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 1/50 2026-01-22 17:12:24,122 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:12:24,122 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 2/50 2026-01-22 17:12:24,605 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:12:24,605 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 3/50 2026-01-22 17:12:25,082 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:12:25,082 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 4/50 2026-01-22 17:12:25,596 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:12:25,596 INFO Evaluation progress: 79% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 5/50 2026-01-22 17:12:26,096 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:12:26,096 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 6/50 2026-01-22 17:12:26,686 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:12:26,686 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 7/50 2026-01-22 17:12:27,149 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:12:27,149 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 8/50 2026-01-22 17:12:27,578 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:12:27,578 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 9/50 2026-01-22 17:12:28,025 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:12:28,025 INFO Evaluation progress: 80% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 10/50 2026-01-22 17:12:28,488 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:12:28,488 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 11/50 2026-01-22 17:12:29,054 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:12:29,054 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 12/50 2026-01-22 17:12:29,615 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:12:29,615 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 13/50 2026-01-22 17:12:30,070 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:12:30,070 INFO Evaluation progress: 81% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 14/50 2026-01-22 17:12:30,526 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:12:30,526 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 15/50 2026-01-22 17:12:31,003 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:12:31,003 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 16/50 2026-01-22 17:12:31,503 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:12:31,503 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 17/50 2026-01-22 17:12:31,973 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:12:31,973 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 18/50 2026-01-22 17:12:32,427 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:12:32,427 INFO Evaluation progress: 82% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 19/50 2026-01-22 17:12:32,886 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:12:32,886 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 20/50 2026-01-22 17:12:33,342 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:12:33,342 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 21/50 2026-01-22 17:12:33,838 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:12:33,838 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 22/50 2026-01-22 17:12:34,292 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:12:34,292 INFO Evaluation progress: 83% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 23/50 2026-01-22 17:12:34,764 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:12:34,764 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 24/50 2026-01-22 17:12:35,280 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:12:35,280 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 25/50 2026-01-22 17:12:35,782 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:12:35,782 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 26/50 2026-01-22 17:12:36,310 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:12:36,310 INFO Evaluation progress: 84% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 27/50 2026-01-22 17:12:36,854 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:12:36,854 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 28/50 2026-01-22 17:12:37,322 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:12:37,322 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 29/50 2026-01-22 17:12:37,781 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:12:37,781 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 30/50 2026-01-22 17:12:38,316 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:12:38,316 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 31/50 2026-01-22 17:12:38,835 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:12:38,835 INFO Evaluation progress: 85% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 32/50 2026-01-22 17:12:39,324 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:12:39,324 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 33/50 2026-01-22 17:12:39,848 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:12:39,848 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 34/50 2026-01-22 17:12:40,334 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:12:40,334 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 35/50 2026-01-22 17:12:40,820 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:12:40,820 INFO Evaluation progress: 86% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 36/50 2026-01-22 17:12:41,303 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:12:41,303 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 37/50 2026-01-22 17:12:41,725 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:12:41,725 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 38/50 2026-01-22 17:12:42,240 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:12:42,240 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 39/50 2026-01-22 17:12:42,763 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:12:42,763 INFO Evaluation progress: 87% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 40/50 2026-01-22 17:12:43,232 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:12:43,232 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 41/50 2026-01-22 17:12:43,677 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:12:43,677 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 42/50 2026-01-22 17:12:44,186 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:12:44,186 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 43/50 2026-01-22 17:12:44,742 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:12:44,742 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 44/50 2026-01-22 17:12:45,168 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:12:45,168 INFO Evaluation progress: 88% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 45/50 2026-01-22 17:12:45,644 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:12:45,644 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 46/50 2026-01-22 17:12:46,198 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:12:46,198 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 47/50 2026-01-22 17:12:46,688 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:12:46,688 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 48/50 2026-01-22 17:12:47,144 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:12:47,144 INFO Evaluation progress: 89% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 49/50 2026-01-22 17:12:47,641 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:12:47,641 INFO Evaluation progress: 90% - Context mean reciprocal rank - Build > config > run 'mean_reciprocal_rank' (cpu) evaluation for input 50/50 2026-01-22 17:12:48,529 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:12:48,529 DEBUG DONE execution of explainer: h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator - created explanations: 4 2026-01-22 17:12:48,530 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:12:48,530 INFO Evaluation progress: 90% - 7/7 evaluator h2o_sonar.evaluators.rag_context_mean_reciprocal_rank_evaluator.MeanReciprocalRankEvaluator run FINISHED 2026-01-22 17:12:48,536 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:12:48,536 INFO Evaluation progress: 90% - Evaluators/explainers run finished 2026-01-22 17:12:48,538 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:12:48,538 INFO Evaluation progress: 91% - Shutting down H2O-3 (if it was started) 2026-01-22 17:12:48,539 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:12:48,539 INFO Evaluation progress: 95% - Finalizing - saving datasets, models, and explanations 2026-01-22 17:12:48,540 INFO Evaluation progress: 100% - DONE 2026-01-22 17:12:48,540 INFO Evaluation progress: 100% - DONE 2026-01-22 17:12:48,546 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:12:48,546 WARNING Unable to index explanations HTML fragment of explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:12:48,564 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html' 2026-01-22 17:12:48,564 WARNING Unable to include explanations HTML fragment for explainer h2o_sonar.evaluators.answer_accuracy_evaluator.AnswerAccuracyEvaluator: [Errno 2] No such file or directory: '/tmp/pytest-of-dvorka/pytest-0/test_evaluate_and_compare__hom5/h2o-sonar/mli_experiment_860a8508-67b0-4112-bc52-5ce11cea1fc4/explainer_h2o_sonar_evaluators_answer_accuracy_evaluator_AnswerAccuracyEvaluator_a71e39d6-75ca-481a-a6c2-4309faff8f4b/global_html_fragment/text_html/explanation.html'