{ "schema_version": "1.0.0", "review_date": "2026-10-02", "analysis_type": "Retrospective recalculation of archived evaluation scores; not a model rerun.", "score_label": "LLM-judge scores, not objective accuracy", "score_scale": { "minimum": 0, "maximum": 100 }, "sample": { "n": 69, "sampling": "Purposively sampled open-source technical tickets; not a random population sample.", "pairing": "Each row contains paired scores for one unique ticket; both runs have the same ticket set.", "case_ids": "Pseudonymous sequential identifiers; no identity lookup is exported. Scores remain linkable if matched to the private source." }, "limitations": [ "Results describe this saved evaluation setting, not general model quality or production accuracy.", "Run08 uses three-way judging; Run09 uses pairwise judging. Do not interpret cross-run score changes as a trend.", "The experiment report describes local Ollama Llama 3 8B; score artifacts do not verify exact model digests, runtime, prompts, or judge settings.", "The inspected evaluator uses a 0-100 rubric, visible method labels and fixed ordering; this export does not establish blind or randomized assessment.", "No ROUGE-L, human accuracy labels, token costs, or controlled latency measurements are included.", "Paired t-tests summarize these saved judge scores; p-values do not establish an architectural mechanism or external generalizability." ], "runs": { "run08": { "label": "Run08: three-way comparison", "source_filename": "evaluation_results_run_08_3way_comparison_ollama_llama3_ollama_llama3.json", "source_sha256": "953f471deaccf01d7314e320e562bd7d31fb0cd7ae4ce6a07b591196ec09d584", "n": 69, "evaluation_protocol": "Three-way LLM judging", "mapping_evidence": "Score fields are interpreted using the Run08 experiment report and evaluator conventions. The JSON has no run-setting metadata.", "settings_evidence_limit": "Model and judge settings are reported context, not independently verified from the archived score JSON.", "methods": { "single_pass": { "label": "Single pass", "source_score_field": "baseline_score", "n": 69, "mean": 79.8840579710145, "sample_sd": 10.249155247663467, "minimum": 0.0, "maximum": 92.0 }, "multi_agent": { "label": "Multi-agent", "source_score_field": "poc_score", "n": 69, "mean": 76.73913043478261, "sample_sd": 14.598199097141928, "minimum": 20.0, "maximum": 95.0 }, "single_pass_cot": { "label": "Single-pass CoT", "source_score_field": "cot_score", "n": 69, "mean": 91.01449275362319, "sample_sd": 6.450017678095219, "minimum": 60.0, "maximum": 100.0 } }, "comparisons": [ { "method_a": "single_pass_cot", "method_b": "multi_agent", "difference_definition": "method_a minus method_b", "n_pairs": 69, "mean_difference": 14.27536231884058, "sample_sd_difference": 12.927267189903189, "t_statistic": 9.172864113313858, "degrees_of_freedom": 68, "p_value_two_sided": 1.647078694497495e-13, "confidence_level": 0.95, "confidence_interval": [ 11.169893662567247, 17.380830975113913 ], "alpha": 0.05, "p_value_adjustment": "none; exploratory comparisons" }, { "method_a": "single_pass_cot", "method_b": "single_pass", "difference_definition": "method_a minus method_b", "n_pairs": 69, "mean_difference": 11.130434782608695, "sample_sd_difference": 11.08511593969226, "t_statistic": 8.340583506106068, "degrees_of_freedom": 68, "p_value_two_sided": 5.275599646613638e-12, "confidence_level": 0.95, "confidence_interval": [ 8.467499175556542, 13.793370389660849 ], "alpha": 0.05, "p_value_adjustment": "none; exploratory comparisons" }, { "method_a": "multi_agent", "method_b": "single_pass", "difference_definition": "method_a minus method_b", "n_pairs": 69, "mean_difference": -3.1449275362318843, "sample_sd_difference": 13.787542717462319, "t_statistic": -1.8947343014593498, "degrees_of_freedom": 68, "p_value_two_sided": 0.06238027418344392, "confidence_level": 0.95, "confidence_interval": [ -6.457056938921946, 0.16720186645817803 ], "alpha": 0.05, "p_value_adjustment": "none; exploratory comparisons" } ], "interpretation": "Single-pass CoT received higher judge scores than the other two methods in this saved three-way evaluation.", "cases": [ { "case_id": "case_001", "scores": { "single_pass": 90.0, "multi_agent": 60.0, "single_pass_cot": 80.0 } }, { "case_id": "case_002", "scores": { "single_pass": 80.0, "multi_agent": 50.0, "single_pass_cot": 60.0 } }, { "case_id": "case_003", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_004", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_005", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_006", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 70.0 } }, { "case_id": "case_007", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_008", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_009", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_010", "scores": { "single_pass": 92.0, "multi_agent": 95.0, "single_pass_cot": 90.0 } }, { "case_id": "case_011", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_012", "scores": { "single_pass": 0.0, "multi_agent": 20.0, "single_pass_cot": 80.0 } }, { "case_id": "case_013", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_014", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_015", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_016", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_017", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_018", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_019", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_020", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_021", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_022", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_023", "scores": { "single_pass": 90.0, "multi_agent": 80.0, "single_pass_cot": 95.0 } }, { "case_id": "case_024", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_025", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_026", "scores": { "single_pass": 90.0, "multi_agent": 60.0, "single_pass_cot": 95.0 } }, { "case_id": "case_027", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_028", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_029", "scores": { "single_pass": 80.0, "multi_agent": 50.0, "single_pass_cot": 90.0 } }, { "case_id": "case_030", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_031", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_032", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_033", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_034", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_035", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_036", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_037", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_038", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_039", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_040", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_041", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_042", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_043", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_044", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_045", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_046", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_047", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_048", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_049", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_050", "scores": { "single_pass": 90.0, "multi_agent": 70.0, "single_pass_cot": 80.0 } }, { "case_id": "case_051", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_052", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_053", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_054", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_055", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_056", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_057", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_058", "scores": { "single_pass": 80.0, "multi_agent": 70.0, "single_pass_cot": 90.0 } }, { "case_id": "case_059", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_060", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_061", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_062", "scores": { "single_pass": 80.0, "multi_agent": 60.0, "single_pass_cot": 90.0 } }, { "case_id": "case_063", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 100.0 } }, { "case_id": "case_064", "scores": { "single_pass": 90.0, "multi_agent": 70.0, "single_pass_cot": 95.0 } }, { "case_id": "case_065", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_066", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } }, { "case_id": "case_067", "scores": { "single_pass": 90.0, "multi_agent": 80.0, "single_pass_cot": 95.0 } }, { "case_id": "case_068", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 70.0 } }, { "case_id": "case_069", "scores": { "single_pass": 80.0, "multi_agent": 90.0, "single_pass_cot": 95.0 } } ] }, "run09": { "label": "Run09: reviewer-loop comparison", "source_filename": "evaluation_results_run_09_paradox_proof.json", "source_sha256": "2d7aa28fbba3143c57359ee52722148304db09eb324365c3c65765cf5f3538ac", "n": 69, "evaluation_protocol": "Pairwise LLM judging", "mapping_evidence": "Source JSON metadata explicitly maps baseline_score to single-pass CoT and poc_score to multi-agent CoT.", "settings_evidence_limit": "Model and judge settings are reported context, not independently verified from the archived score JSON.", "methods": { "single_pass_cot": { "label": "Single-pass CoT", "source_score_field": "baseline_score", "n": 69, "mean": 87.8840579710145, "sample_sd": 13.573911213929785, "minimum": 0.0, "maximum": 95.0 }, "multi_agent_cot": { "label": "CoT + reviewer", "source_score_field": "poc_score", "n": 69, "mean": 86.98550724637681, "sample_sd": 4.529292743202098, "minimum": 80.0, "maximum": 98.0 } }, "comparisons": [ { "method_a": "multi_agent_cot", "method_b": "single_pass_cot", "difference_definition": "method_a minus method_b", "n_pairs": 69, "mean_difference": -0.8985507246376812, "sample_sd_difference": 15.121220896413242, "t_statistic": -0.4936058366218389, "degrees_of_freedom": 68, "p_value_two_sided": 0.6231751230188197, "confidence_level": 0.95, "confidence_interval": [ -4.531064607778696, 2.7339631585033337 ], "alpha": 0.05, "p_value_adjustment": "none; exploratory comparisons" } ], "interpretation": "The reviewer condition had a slightly lower observed mean; the paired difference was not statistically significant. This does not prove degradation or equivalence.", "cases": [ { "case_id": "case_001", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_002", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_003", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_004", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_005", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_006", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_007", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_008", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_009", "scores": { "single_pass_cot": 90.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_010", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_011", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_012", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_013", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_014", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_015", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_016", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_017", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_018", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_019", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_020", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_021", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_022", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_023", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_024", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_025", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_026", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_027", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_028", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_029", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_030", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_031", "scores": { "single_pass_cot": 79.0, "multi_agent_cot": 95.0 } }, { "case_id": "case_032", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_033", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_034", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_035", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_036", "scores": { "single_pass_cot": 90.0, "multi_agent_cot": 95.0 } }, { "case_id": "case_037", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_038", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_039", "scores": { "single_pass_cot": 79.0, "multi_agent_cot": 92.0 } }, { "case_id": "case_040", "scores": { "single_pass_cot": 90.0, "multi_agent_cot": 95.0 } }, { "case_id": "case_041", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_042", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_043", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_044", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_045", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_046", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_047", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_048", "scores": { "single_pass_cot": 0.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_049", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_050", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_051", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_052", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_053", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_054", "scores": { "single_pass_cot": 85.0, "multi_agent_cot": 92.0 } }, { "case_id": "case_055", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_056", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_057", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_058", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_059", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 80.0 } }, { "case_id": "case_060", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_061", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } }, { "case_id": "case_062", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_063", "scores": { "single_pass_cot": 80.0, "multi_agent_cot": 95.0 } }, { "case_id": "case_064", "scores": { "single_pass_cot": 85.0, "multi_agent_cot": 95.0 } }, { "case_id": "case_065", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 98.0 } }, { "case_id": "case_066", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_067", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_068", "scores": { "single_pass_cot": 51.0, "multi_agent_cot": 90.0 } }, { "case_id": "case_069", "scores": { "single_pass_cot": 95.0, "multi_agent_cot": 85.0 } } ] } } }