{"scenario_id": "pro-bare", "category": "bare", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "I don't have information on the iPhone 18 Pro, as my training data only goes up to 2023, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.088, "eval_s": 1.258, "tokens_per_s": 31.8, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price in US dollars of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 3.098, "eval_s": 0.538, "tokens_per_s": 35.32, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.73, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 1.232, "eval_s": 0.72, "tokens_per_s": 34.72, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.04, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 2.405, "eval_s": 0.538, "tokens_per_s": 35.31, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.0, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 1.517, "eval_s": 0.461, "tokens_per_s": 34.73, "eval_tokens": 16, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.08, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "I don't have information on the iPhone 18 Pro Max as my training data only goes up to 2022, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.271, "eval_s": 1.166, "tokens_per_s": 34.31, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.6, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price in US dollars of the iPhone 18 Pro Max is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.589, "eval_s": 0.563, "tokens_per_s": 35.53, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.22, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399, according to a report by TrendForce six days before Apple's event.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.832, "eval_s": 1.153, "tokens_per_s": 34.69, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 1.748, "eval_s": 1.05, "tokens_per_s": 34.29, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.83, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months, according to Apple's press release.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 1.232, "eval_s": 1.265, "tokens_per_s": 34.0, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "I don't have information on the specific pricing of Apple's foldable iPhone Duo in the United States, as my training data may not be up-to-date or may not include this specific product.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.793, "eval_s": 1.168, "tokens_per_s": 34.26, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 for the base model, with prices increasing for larger storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.859, "eval_s": 0.978, "tokens_per_s": 33.73, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is expected to be between $2,099 and $2,299, according to a research firm's report.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 1.096, "eval_s": 1.076, "tokens_per_s": 34.38, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.29, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.674, "eval_s": 0.612, "tokens_per_s": 34.32, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.34, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999, with a monthly payment option of $83.29 for 24 months.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.671, "eval_s": 1.058, "tokens_per_s": 34.03, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.87, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US, as announced by Apple, is $1,199, which is lower than the expected price range of $1,249 to $1,299 mentioned by the research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 5.315, "eval_s": 1.76, "tokens_per_s": 32.96, "eval_tokens": 58, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 9.24, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US, as announced by Apple, is $1,299, which is higher than the claimed range of $1,349 to $1,399.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 3.976, "eval_s": 1.539, "tokens_per_s": 33.14, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 7.62, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone Duo in the US, as announced by Apple, was $1,999, which is lower than the claimed price range of $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 3.816, "eval_s": 1.48, "tokens_per_s": 33.12, "eval_tokens": 49, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 7.42, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe claim made before the event stated that the 2026 foldable iPhone would start at $2,399, but Apple announced that the iPhone Duo, which is not a foldable iPhone, starts at $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.086, "eval_s": 1.686, "tokens_per_s": 33.21, "eval_tokens": 56, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.0, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US was $1,199, which is a $100 increase from the starting price of the iPhone 17 Pro.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 3.256, "eval_s": 1.288, "tokens_per_s": 33.38, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.71, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US was $1,299, which is $100 more than the claimed increase.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 3.33, "eval_s": 1.108, "tokens_per_s": 33.39, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.59, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "I don't have information on the iPhone 18 Pro, as my training data only goes up to 2023, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.082, "eval_s": 1.232, "tokens_per_s": 32.46, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "I don't have information on the iPhone 18 Pro, as my training data only goes up to 2023, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.082, "eval_s": 1.239, "tokens_per_s": 32.28, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.38, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "I don't have information on the iPhone 18 Pro, as my training data only goes up to 2023, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.075, "eval_s": 1.261, "tokens_per_s": 31.71, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "I don't have information on the iPhone 18 Pro, as my training data only goes up to 2023, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.095, "eval_s": 1.258, "tokens_per_s": 31.79, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price in US dollars of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 2.708, "eval_s": 0.536, "tokens_per_s": 35.46, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.32, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price in US dollars of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.083, "eval_s": 0.601, "tokens_per_s": 31.61, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.76, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price in US dollars of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.092, "eval_s": 0.65, "tokens_per_s": 29.21, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price in US dollars of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.091, "eval_s": 0.638, "tokens_per_s": 29.8, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.82, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 1.191, "eval_s": 0.725, "tokens_per_s": 34.46, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a report by TrendForce six days before Apple's event.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.079, "eval_s": 1.247, "tokens_per_s": 31.28, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a report by TrendForce six days before Apple's event.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.082, "eval_s": 1.227, "tokens_per_s": 31.79, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a report by TrendForce six days before Apple's event.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.087, "eval_s": 1.234, "tokens_per_s": 31.6, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.4, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 1.553, "eval_s": 0.56, "tokens_per_s": 33.94, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.16, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.072, "eval_s": 0.627, "tokens_per_s": 30.3, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.76, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.082, "eval_s": 0.568, "tokens_per_s": 33.44, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.082, "eval_s": 0.622, "tokens_per_s": 30.55, "eval_tokens": 19, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 1.34, "eval_s": 0.447, "tokens_per_s": 35.77, "eval_tokens": 16, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.9, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.075, "eval_s": 0.553, "tokens_per_s": 28.93, "eval_tokens": 16, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.67, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.085, "eval_s": 0.522, "tokens_per_s": 30.67, "eval_tokens": 16, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.085, "eval_s": 0.548, "tokens_per_s": 29.21, "eval_tokens": 16, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.69, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "I don't have information on the iPhone 18 Pro Max as my training data only goes up to 2022, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.248, "eval_s": 1.161, "tokens_per_s": 34.45, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.54, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "I don't have information on the iPhone 18 Pro Max as my training data only goes up to 2022, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.087, "eval_s": 1.265, "tokens_per_s": 31.62, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "I don't have information on the iPhone 18 Pro Max as my training data only goes up to 2022, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.085, "eval_s": 1.164, "tokens_per_s": 34.36, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.31, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "I don't have information on the iPhone 18 Pro Max as my training data only goes up to 2022, and I couldn't find any information on an iPhone model with that name.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.068, "eval_s": 1.239, "tokens_per_s": 32.28, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.34, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price in US dollars of the iPhone 18 Pro Max is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.646, "eval_s": 0.521, "tokens_per_s": 38.36, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.26, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price in US dollars of the iPhone 18 Pro Max is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.076, "eval_s": 0.637, "tokens_per_s": 31.4, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price in US dollars of the iPhone 18 Pro Max is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.084, "eval_s": 0.632, "tokens_per_s": 31.65, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price in US dollars of the iPhone 18 Pro Max is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.083, "eval_s": 0.633, "tokens_per_s": 31.59, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.8, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399, according to a report by TrendForce six days before Apple's event.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.62, "eval_s": 1.169, "tokens_per_s": 34.21, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.91, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.086, "eval_s": 0.826, "tokens_per_s": 31.48, "eval_tokens": 26, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.98, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.086, "eval_s": 0.798, "tokens_per_s": 32.56, "eval_tokens": 26, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.082, "eval_s": 0.825, "tokens_per_s": 31.51, "eval_tokens": 26, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 1.489, "eval_s": 1.044, "tokens_per_s": 34.5, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.59, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.084, "eval_s": 1.091, "tokens_per_s": 33.01, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.085, "eval_s": 1.061, "tokens_per_s": 33.92, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.19, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.084, "eval_s": 1.096, "tokens_per_s": 32.86, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.25, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months, according to Apple's press release.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 1.17, "eval_s": 1.266, "tokens_per_s": 33.97, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.54, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months, according to Apple's press release.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.07, "eval_s": 1.356, "tokens_per_s": 31.72, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.48, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months, according to Apple's press release.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.085, "eval_s": 1.32, "tokens_per_s": 32.58, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.47, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) or $54.12 (U.S.) per month for 24 months, according to Apple's press release.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.093, "eval_s": 1.344, "tokens_per_s": 32.0, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.49, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "I don't have information on the specific pricing of Apple's foldable iPhone Duo in the United States, as my training data may not be up-to-date or may not include this specific product.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.542, "eval_s": 1.183, "tokens_per_s": 33.8, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.86, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "I don't have information on the specific pricing of Apple's foldable iPhone Duo in the United States, as my training data may not be up-to-date or may not include this specific product.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.076, "eval_s": 1.243, "tokens_per_s": 32.17, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.37, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "I don't have information on the specific pricing of Apple's foldable iPhone Duo in the United States, as my training data may not be up-to-date or may not include this specific product.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.085, "eval_s": 1.271, "tokens_per_s": 31.47, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "I don't have information on the specific pricing of Apple's foldable iPhone Duo in the United States, as my training data may not be up-to-date or may not include this specific product.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.076, "eval_s": 1.265, "tokens_per_s": 31.61, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.38, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 for the base model, with prices increasing for larger storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.216, "eval_s": 0.984, "tokens_per_s": 33.52, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.34, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 for the base model, with prices increasing for larger storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.081, "eval_s": 1.057, "tokens_per_s": 31.23, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.2, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 for the base model, with prices increasing for larger storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.071, "eval_s": 1.059, "tokens_per_s": 31.17, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.18, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 for the base model, with prices increasing for larger storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.075, "eval_s": 1.058, "tokens_per_s": 31.18, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.17, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is expected to be between $2,099 and $2,299, according to a research firm's report.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.852, "eval_s": 1.095, "tokens_per_s": 33.8, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is expected to be between $2,099 and $2,299, according to a research firm's report.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.089, "eval_s": 1.16, "tokens_per_s": 31.91, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.31, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is expected to be between $2,099 and $2,299, according to a research firm's report.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.081, "eval_s": 1.169, "tokens_per_s": 31.66, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.3, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is expected to be between $2,099 and $2,299, according to a research firm's report.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.084, "eval_s": 1.178, "tokens_per_s": 31.42, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.32, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.424, "eval_s": 0.604, "tokens_per_s": 34.74, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.079, "eval_s": 0.696, "tokens_per_s": 30.17, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.82, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.065, "eval_s": 0.566, "tokens_per_s": 37.11, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.67, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.073, "eval_s": 0.579, "tokens_per_s": 36.28, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.73, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999, with a monthly payment option of $83.29 for 24 months.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.473, "eval_s": 0.724, "tokens_per_s": 49.7, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.28, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999, with a monthly payment option of $83.29 for 24 months.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.052, "eval_s": 0.855, "tokens_per_s": 42.09, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.94, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999, with a monthly payment option of $83.29 for 24 months.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.055, "eval_s": 0.876, "tokens_per_s": 41.1, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999, with a monthly payment option of $83.29 for 24 months.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.068, "eval_s": 0.895, "tokens_per_s": 40.22, "eval_tokens": 36, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.0, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US, as announced by Apple, is $1,199, which is lower than the expected price range of $1,249 to $1,299 mentioned by the research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 5.0, "eval_s": 1.238, "tokens_per_s": 46.84, "eval_tokens": 58, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 8.34, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US, as announced by Apple, is $1,199, which is lower than the expected price range of $1,249 to $1,299 mentioned by the research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.058, "eval_s": 1.339, "tokens_per_s": 43.31, "eval_tokens": 58, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.44, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US, as announced by Apple, is $1,199, which is lower than the expected price range of $1,249 to $1,299 mentioned by the research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.058, "eval_s": 1.374, "tokens_per_s": 42.21, "eval_tokens": 58, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.46, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US, as announced by Apple, is $1,199, which is lower than the expected price range of $1,249 to $1,299 mentioned by the research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.075, "eval_s": 1.282, "tokens_per_s": 45.23, "eval_tokens": 58, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US, as announced by Apple, is $1,299, which is higher than the claimed range of $1,349 to $1,399.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 2.852, "eval_s": 1.046, "tokens_per_s": 48.75, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US, as announced by Apple, is $1,299, which is higher than the claimed range of $1,349 to $1,399.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.067, "eval_s": 1.209, "tokens_per_s": 42.18, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.31, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US, as announced by Apple, is $1,299, which is higher than the claimed range of $1,349 to $1,399.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.08, "eval_s": 1.202, "tokens_per_s": 42.44, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.33, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US, as announced by Apple, is $1,299, which is higher than the claimed range of $1,349 to $1,399.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.067, "eval_s": 1.191, "tokens_per_s": 42.83, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.28, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone Duo in the US, as announced by Apple, was $1,999, which is lower than the claimed price range of $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.931, "eval_s": 1.007, "tokens_per_s": 48.65, "eval_tokens": 49, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.02, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone Duo in the US, as announced by Apple, was $1,999, which is lower than the claimed price range of $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.076, "eval_s": 1.092, "tokens_per_s": 44.87, "eval_tokens": 49, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.22, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone Duo in the US, as announced by Apple, was $1,999, which is lower than the claimed price range of $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.087, "eval_s": 1.088, "tokens_per_s": 45.04, "eval_tokens": 49, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone Duo in the US, as announced by Apple, was $1,999, which is lower than the claimed price range of $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.076, "eval_s": 1.073, "tokens_per_s": 45.67, "eval_tokens": 49, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.22, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe claim made before the event stated that the 2026 foldable iPhone would start at $2,399, but Apple announced that the iPhone Duo, which is not a foldable iPhone, starts at $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 1.855, "eval_s": 1.153, "tokens_per_s": 48.58, "eval_tokens": 56, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.11, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe claim made before the event stated that the 2026 foldable iPhone would start at $2,399, but Apple announced that the iPhone Duo, which is not a foldable iPhone, starts at $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.065, "eval_s": 1.307, "tokens_per_s": 42.86, "eval_tokens": 56, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe claim made before the event stated that the 2026 foldable iPhone would start at $2,399, but Apple announced that the iPhone Duo, which is not a foldable iPhone, starts at $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.075, "eval_s": 1.305, "tokens_per_s": 42.9, "eval_tokens": 56, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.44, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe claim made before the event stated that the 2026 foldable iPhone would start at $2,399, but Apple announced that the iPhone Duo, which is not a foldable iPhone, starts at $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.059, "eval_s": 1.301, "tokens_per_s": 43.03, "eval_tokens": 56, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US was $1,199, which is a $100 increase from the starting price of the iPhone 17 Pro.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 2.784, "eval_s": 0.876, "tokens_per_s": 49.11, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US was $1,199, which is a $100 increase from the starting price of the iPhone 17 Pro.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.077, "eval_s": 0.988, "tokens_per_s": 43.52, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.1, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US was $1,199, which is a $100 increase from the starting price of the iPhone 17 Pro.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.06, "eval_s": 1.024, "tokens_per_s": 41.99, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.11, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro in the US was $1,199, which is a $100 increase from the starting price of the iPhone 17 Pro.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.077, "eval_s": 1.006, "tokens_per_s": 42.74, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.12, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US was $1,299, which is $100 more than the claimed increase.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 2.807, "eval_s": 0.748, "tokens_per_s": 49.46, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US was $1,299, which is $100 more than the claimed increase.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.062, "eval_s": 0.94, "tokens_per_s": 39.35, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US was $1,299, which is $100 more than the claimed increase.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.062, "eval_s": 0.905, "tokens_per_s": 40.89, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "llama3.2:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price of the iPhone 18 Pro Max in the US was $1,299, which is $100 more than the claimed increase.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.058, "eval_s": 0.916, "tokens_per_s": 40.39, "eval_tokens": 37, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.0, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "I do not know the starting price of the iPhone 18 Pro as it does not exist and I was not trained with this specific product information.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.059, "eval_s": 0.723, "tokens_per_s": 42.86, "eval_tokens": 31, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "I do not know the starting price of the iPhone 18 Pro as it does not exist and I was not trained with this specific product information.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.055, "eval_s": 0.721, "tokens_per_s": 43.0, "eval_tokens": 31, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "I do not know the starting price of the iPhone 18 Pro as it does not exist and I was not trained with this specific product information.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.059, "eval_s": 0.718, "tokens_per_s": 43.16, "eval_tokens": 31, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "I do not know the starting price of the iPhone 18 Pro as it does not exist and I was not trained with this specific product information.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.062, "eval_s": 0.712, "tokens_per_s": 43.52, "eval_tokens": 31, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "I do not know the starting price of the iPhone 18 Pro as it does not exist and I was not trained with this specific product information.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.053, "eval_s": 0.737, "tokens_per_s": 42.08, "eval_tokens": 31, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro in US dollars is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.701, "eval_s": 0.399, "tokens_per_s": 55.12, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.15, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro in US dollars is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.062, "eval_s": 0.545, "tokens_per_s": 40.38, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.64, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro in US dollars is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.058, "eval_s": 0.557, "tokens_per_s": 39.47, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.67, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro in US dollars is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.064, "eval_s": 0.532, "tokens_per_s": 41.35, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.65, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro in US dollars is $1,199.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.053, "eval_s": 0.515, "tokens_per_s": 42.68, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.61, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro, according to a MacRumors report, is expected to be between $1,249 and $1,299, as predicted by research firm TrendForce six days before Apple's event in 2026.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.489, "eval_s": 1.145, "tokens_per_s": 52.39, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.7, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro, according to a MacRumors report, is expected to be between $1,249 and $1,299, as predicted by research firm TrendForce six days before Apple's event in 2026.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.06, "eval_s": 1.274, "tokens_per_s": 47.11, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro, according to a MacRumors report, is expected to be between $1,249 and $1,299, as predicted by research firm TrendForce six days before Apple's event in 2026.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.065, "eval_s": 1.184, "tokens_per_s": 50.69, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.3, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro, according to a MacRumors report, is expected to be between $1,249 and $1,299, as predicted by research firm TrendForce six days before Apple's event in 2026.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.053, "eval_s": 1.311, "tokens_per_s": 45.77, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro, according to a MacRumors report, is expected to be between $1,249 and $1,299, as predicted by research firm TrendForce six days before Apple's event in 2026.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.063, "eval_s": 1.268, "tokens_per_s": 47.32, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier predicted a higher starting price of between $1,249 and $1,299.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.578, "eval_s": 1.128, "tokens_per_s": 53.19, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier predicted a higher starting price of between $1,249 and $1,299.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.073, "eval_s": 1.26, "tokens_per_s": 47.61, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.37, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier predicted a higher starting price of between $1,249 and $1,299.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.055, "eval_s": 1.268, "tokens_per_s": 47.33, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.36, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier predicted a higher starting price of between $1,249 and $1,299.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.06, "eval_s": 1.283, "tokens_per_s": 46.77, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.4, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier predicted a higher starting price of between $1,249 and $1,299.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.062, "eval_s": 1.294, "tokens_per_s": 46.38, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.), as stated in Apple's press release of 9 September 2026. However, there is conflicting information suggesting that the starting price might be expected to be between $1,249 and $1,299 according to a MacRumors report and research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.521, "eval_s": 1.58, "tokens_per_s": 52.52, "eval_tokens": 83, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.15, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.), as stated in Apple's press release of 9 September 2026. However, there is conflicting information suggesting that the starting price might be expected to be between $1,249 and $1,299 according to a MacRumors report and research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.059, "eval_s": 1.723, "tokens_per_s": 48.17, "eval_tokens": 83, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.82, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.), as stated in Apple's press release of 9 September 2026. However, there is conflicting information suggesting that the starting price might be expected to be between $1,249 and $1,299 according to a MacRumors report and research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.079, "eval_s": 1.727, "tokens_per_s": 48.07, "eval_tokens": 83, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.85, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.), as stated in Apple's press release of 9 September 2026. However, there is conflicting information suggesting that the starting price might be expected to be between $1,249 and $1,299 according to a MacRumors report and research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.066, "eval_s": 1.673, "tokens_per_s": 49.6, "eval_tokens": 83, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.8, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.), as stated in Apple's press release of 9 September 2026. However, there is conflicting information suggesting that the starting price might be expected to be between $1,249 and $1,299 according to a MacRumors report and research firm TrendForce.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.055, "eval_s": 1.701, "tokens_per_s": 48.81, "eval_tokens": 83, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.8, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "I do not know the starting price of the iPhone 18 Pro Max as it is not a real product and it does not exist. My knowledge ends with the latest data available before my training cutoff in 2021.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.22, "eval_s": 0.91, "tokens_per_s": 52.72, "eval_tokens": 48, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "I do not know the starting price of the iPhone 18 Pro Max as it is not a real product and it does not exist. My knowledge ends with the latest data available before my training cutoff in 2021.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.068, "eval_s": 1.019, "tokens_per_s": 47.09, "eval_tokens": 48, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.14, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "I do not know the starting price of the iPhone 18 Pro Max as it is not a real product and it does not exist. My knowledge ends with the latest data available before my training cutoff in 2021.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.06, "eval_s": 1.039, "tokens_per_s": 46.18, "eval_tokens": 48, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.15, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "I do not know the starting price of the iPhone 18 Pro Max as it is not a real product and it does not exist. My knowledge ends with the latest data available before my training cutoff in 2021.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.06, "eval_s": 1.005, "tokens_per_s": 47.76, "eval_tokens": 48, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.1, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "I do not know the starting price of the iPhone 18 Pro Max as it is not a real product and it does not exist. My knowledge ends with the latest data available before my training cutoff in 2021.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.059, "eval_s": 1.022, "tokens_per_s": 46.98, "eval_tokens": 48, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.11, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.319, "eval_s": 0.424, "tokens_per_s": 54.27, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.052, "eval_s": 0.566, "tokens_per_s": 40.63, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.64, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.059, "eval_s": 0.547, "tokens_per_s": 42.08, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.063, "eval_s": 0.512, "tokens_per_s": 44.96, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.072, "eval_s": 0.487, "tokens_per_s": 47.27, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.62, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max, according to the MacRumors report, is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.407, "eval_s": 0.784, "tokens_per_s": 52.28, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.26, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max, according to the MacRumors report, is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.06, "eval_s": 0.906, "tokens_per_s": 45.24, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max, according to the MacRumors report, is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.064, "eval_s": 0.91, "tokens_per_s": 45.08, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.03, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max, according to the MacRumors report, is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.062, "eval_s": 0.904, "tokens_per_s": 45.38, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max, according to the MacRumors report, is expected to be between $1,349 and $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.065, "eval_s": 0.904, "tokens_per_s": 45.34, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.0, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier expected the iPhone 18 Pro Max to start between $1,349 and $1,399, which contradicts the official announcement.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.547, "eval_s": 1.383, "tokens_per_s": 52.05, "eval_tokens": 72, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier expected the iPhone 18 Pro Max to start between $1,349 and $1,399, which contradicts the official announcement.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.063, "eval_s": 1.459, "tokens_per_s": 49.34, "eval_tokens": 72, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.57, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier expected the iPhone 18 Pro Max to start between $1,349 and $1,399, which contradicts the official announcement.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.055, "eval_s": 1.52, "tokens_per_s": 47.38, "eval_tokens": 72, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.61, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier expected the iPhone 18 Pro Max to start between $1,349 and $1,399, which contradicts the official announcement.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.053, "eval_s": 1.511, "tokens_per_s": 47.65, "eval_tokens": 72, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.61, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max, according to Apple's press release, is $1,199. However, the MacRumors report from six days earlier expected the iPhone 18 Pro Max to start between $1,349 and $1,399, which contradicts the official announcement.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.054, "eval_s": 1.516, "tokens_per_s": 47.49, "eval_tokens": 72, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.61, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,199 according to Apple's press release, while a MacRumors report suggests it could start between $1,349 and $1,399. There is no single definitive starting price provided in the given information.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.542, "eval_s": 1.251, "tokens_per_s": 51.95, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.86, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,199 according to Apple's press release, while a MacRumors report suggests it could start between $1,349 and $1,399. There is no single definitive starting price provided in the given information.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.065, "eval_s": 1.358, "tokens_per_s": 47.88, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.48, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,199 according to Apple's press release, while a MacRumors report suggests it could start between $1,349 and $1,399. There is no single definitive starting price provided in the given information.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.061, "eval_s": 1.334, "tokens_per_s": 48.74, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.43, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,199 according to Apple's press release, while a MacRumors report suggests it could start between $1,349 and $1,399. There is no single definitive starting price provided in the given information.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.078, "eval_s": 1.33, "tokens_per_s": 48.88, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.46, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,199 according to Apple's press release, while a MacRumors report suggests it could start between $1,349 and $1,399. There is no single definitive starting price provided in the given information.", "parsed": 1199.0, "expected": 1299.0, "correct": false, "ttft_s": 0.063, "eval_s": 1.386, "tokens_per_s": 46.9, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.48, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "I do not know the cost of Apple's foldable iPhone Duo as it is not a product that exists yet, and I don't have information about future products.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.329, "eval_s": 0.637, "tokens_per_s": 53.36, "eval_tokens": 34, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "I do not have information about the cost of Apple's foldable iPhone Duo in the United States, as this product is not yet available and I do not have details about its pricing.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.052, "eval_s": 0.857, "tokens_per_s": 44.31, "eval_tokens": 38, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "I do not have information about the cost of Apple's foldable iPhone Duo in the United States, as this product is not yet available and I do not have details about its pricing.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.059, "eval_s": 0.868, "tokens_per_s": 43.8, "eval_tokens": 38, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.96, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "I do not have information about the cost of Apple's foldable iPhone Duo in the United States, as this product is not yet available and I do not have details about its pricing.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.082, "eval_s": 0.771, "tokens_per_s": 49.28, "eval_tokens": 38, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.9, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "I do not have information about the cost of Apple's foldable iPhone Duo in the United States, as this product is not yet available and I do not have details about its pricing.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.068, "eval_s": 0.853, "tokens_per_s": 44.56, "eval_tokens": 38, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.96, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States and can also be purchased with a monthly subscription plan of $83.29 for 24 months for 256GB, 512GB, 1TB, and 2TB storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.569, "eval_s": 1.235, "tokens_per_s": 52.63, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.88, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States and can also be purchased with a monthly subscription plan of $83.29 for 24 months for 256GB, 512GB, 1TB, and 2TB storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.058, "eval_s": 1.39, "tokens_per_s": 46.77, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.49, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States and can also be purchased with a monthly subscription plan of $83.29 for 24 months for 256GB, 512GB, 1TB, and 2TB storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.054, "eval_s": 1.345, "tokens_per_s": 48.33, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.45, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States and can also be purchased with a monthly subscription plan of $83.29 for 24 months for 256GB, 512GB, 1TB, and 2TB storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.055, "eval_s": 1.368, "tokens_per_s": 47.52, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.48, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States and can also be purchased with a monthly subscription plan of $83.29 for 24 months for 256GB, 512GB, 1TB, and 2TB storage capacities.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.062, "eval_s": 1.312, "tokens_per_s": 49.55, "eval_tokens": 65, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The MacRumors report, citing research firm TrendForce, expects Apple's first foldable iPhone, the iPhone Duo, to start at a price range of $2,099 to $2,299 in the United States.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.429, "eval_s": 0.964, "tokens_per_s": 52.92, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.46, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The MacRumors report, citing research firm TrendForce, expects Apple's first foldable iPhone, the iPhone Duo, to start at a price range of $2,099 to $2,299 in the United States.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.057, "eval_s": 1.096, "tokens_per_s": 46.52, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.19, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The MacRumors report, citing research firm TrendForce, expects Apple's first foldable iPhone, the iPhone Duo, to start at a price range of $2,099 to $2,299 in the United States.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.063, "eval_s": 1.109, "tokens_per_s": 46.0, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.23, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The MacRumors report, citing research firm TrendForce, expects Apple's first foldable iPhone, the iPhone Duo, to start at a price range of $2,099 to $2,299 in the United States.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.062, "eval_s": 1.115, "tokens_per_s": 45.74, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.24, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The MacRumors report, citing research firm TrendForce, expects Apple's first foldable iPhone, the iPhone Duo, to start at a price range of $2,099 to $2,299 in the United States.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.062, "eval_s": 1.087, "tokens_per_s": 46.92, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.18, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 and can be paid for with a monthly subscription of $83.29 for 24 months, depending on the storage capacity. However, the MacRumors report from six days prior suggested the device could start at $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.477, "eval_s": 1.573, "tokens_per_s": 52.13, "eval_tokens": 82, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 and can be paid for with a monthly subscription of $83.29 for 24 months, depending on the storage capacity. However, the MacRumors report from six days prior suggested the device could start at $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.062, "eval_s": 1.733, "tokens_per_s": 47.33, "eval_tokens": 82, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 and can be paid for with a monthly subscription of $83.29 for 24 months, depending on the storage capacity. However, the MacRumors report from six days prior suggested the device could start at $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.056, "eval_s": 1.715, "tokens_per_s": 47.82, "eval_tokens": 82, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 and can be paid for with a monthly subscription of $83.29 for 24 months, depending on the storage capacity. However, the MacRumors report from six days prior suggested the device could start at $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.055, "eval_s": 1.735, "tokens_per_s": 47.25, "eval_tokens": 82, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.83, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States starts at $1,999 and can be paid for with a monthly subscription of $83.29 for 24 months, depending on the storage capacity. However, the MacRumors report from six days prior suggested the device could start at $2,099 to $2,299.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.081, "eval_s": 1.706, "tokens_per_s": 48.07, "eval_tokens": 82, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "The press release from Apple on 9 September 2026 indicates that the iPhone Duo starts at $1,999 in the United States. However, a MacRumors report from six days earlier suggested that the iPhone Duo could start at anywhere between $2,099 and $2,299. Therefore, the actual cost of the iPhone Duo in the United States is not definitively known from the given information.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.513, "eval_s": 1.763, "tokens_per_s": 52.19, "eval_tokens": 92, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.37, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "The press release from Apple on 9 September 2026 indicates that the iPhone Duo starts at $1,999 in the United States. However, a MacRumors report from six days earlier suggested that the iPhone Duo could start at anywhere between $2,099 and $2,299. Therefore, the actual cost of the iPhone Duo in the United States is not definitively known from the given information.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.062, "eval_s": 1.911, "tokens_per_s": 48.14, "eval_tokens": 92, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.04, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "The press release from Apple on 9 September 2026 indicates that the iPhone Duo starts at $1,999 in the United States. However, a MacRumors report from six days earlier suggested that the iPhone Duo could start at anywhere between $2,099 and $2,299. Therefore, the actual cost of the iPhone Duo in the United States is not definitively known from the given information.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.056, "eval_s": 1.914, "tokens_per_s": 48.07, "eval_tokens": 92, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.02, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "The press release from Apple on 9 September 2026 indicates that the iPhone Duo starts at $1,999 in the United States. However, a MacRumors report from six days earlier suggested that the iPhone Duo could start at anywhere between $2,099 and $2,299. Therefore, the actual cost of the iPhone Duo in the United States is not definitively known from the given information.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.055, "eval_s": 1.921, "tokens_per_s": 47.89, "eval_tokens": 92, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "The press release from Apple on 9 September 2026 indicates that the iPhone Duo starts at $1,999 in the United States. However, a MacRumors report from six days earlier suggested that the iPhone Duo could start at anywhere between $2,099 and $2,299. Therefore, the actual cost of the iPhone Duo in the United States is not definitively known from the given information.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.063, "eval_s": 1.901, "tokens_per_s": 48.38, "eval_tokens": 92, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.02, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is significantly lower than the expected starting price of $1,249 to $1,299 as reported by MacRumors.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.733, "eval_s": 1.192, "tokens_per_s": 51.17, "eval_tokens": 61, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is significantly lower than the expected starting price of $1,249 to $1,299 as reported by MacRumors.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.071, "eval_s": 1.323, "tokens_per_s": 46.11, "eval_tokens": 61, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.44, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is significantly lower than the expected starting price of $1,249 to $1,299 as reported by MacRumors.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.058, "eval_s": 1.317, "tokens_per_s": 46.32, "eval_tokens": 61, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.43, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is significantly lower than the expected starting price of $1,249 to $1,299 as reported by MacRumors.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.058, "eval_s": 1.318, "tokens_per_s": 46.27, "eval_tokens": 61, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.44, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is significantly lower than the expected starting price of $1,249 to $1,299 as reported by MacRumors.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.06, "eval_s": 1.302, "tokens_per_s": 46.84, "eval_tokens": 61, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is higher than the expected starting price of $1,349 to $1,399 mentioned in the claim.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.719, "eval_s": 1.158, "tokens_per_s": 51.83, "eval_tokens": 60, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.94, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is higher than the expected starting price of $1,349 to $1,399 as reported by MacRumors.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.075, "eval_s": 1.369, "tokens_per_s": 46.01, "eval_tokens": 63, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.48, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is higher than the expected starting price of $1,349 to $1,399 as reported by MacRumors.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.057, "eval_s": 1.357, "tokens_per_s": 46.42, "eval_tokens": 63, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.47, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is higher than the expected starting price of $1,349 to $1,399 as reported by MacRumors.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.073, "eval_s": 1.337, "tokens_per_s": 47.1, "eval_tokens": 63, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.46, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is higher than the expected starting price of $1,349 to $1,399 as reported by MacRumors.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.059, "eval_s": 1.309, "tokens_per_s": 48.14, "eval_tokens": 63, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.4, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999, which is significantly lower than the $2,099 to $2,299 range expected by TrendForce.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.689, "eval_s": 0.974, "tokens_per_s": 52.34, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.72, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999, which is significantly lower than the $2,099 to $2,299 range expected by TrendForce.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.07, "eval_s": 1.131, "tokens_per_s": 45.11, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.24, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999, which is significantly lower than the $2,099 to $2,299 range expected by TrendForce.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.06, "eval_s": 1.122, "tokens_per_s": 45.44, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999, which is significantly lower than the $2,099 to $2,299 range expected by TrendForce.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.067, "eval_s": 1.148, "tokens_per_s": 44.43, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.24, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999, which is significantly lower than the $2,099 to $2,299 range expected by TrendForce.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.059, "eval_s": 1.118, "tokens_per_s": 45.63, "eval_tokens": 51, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.2, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999 (U.S.), not $2,399 as claimed in the TechRepublic report.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.712, "eval_s": 0.87, "tokens_per_s": 50.56, "eval_tokens": 44, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.65, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999 (U.S.), not $2,399 as claimed in the TechRepublic report.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.055, "eval_s": 1.002, "tokens_per_s": 43.93, "eval_tokens": 44, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.1, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999 (U.S.), not $2,399 as claimed in the TechRepublic report.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.072, "eval_s": 0.931, "tokens_per_s": 47.26, "eval_tokens": 44, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999 (U.S.), not $2,399 as claimed in the TechRepublic report.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.069, "eval_s": 0.903, "tokens_per_s": 48.73, "eval_tokens": 44, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price announced by Apple for the iPhone Duo was $1,999 (U.S.), not $2,399 as claimed in the TechRepublic report.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.074, "eval_s": 0.993, "tokens_per_s": 44.32, "eval_tokens": 44, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.12, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is $100 more than the claim made before the event.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.699, "eval_s": 0.857, "tokens_per_s": 52.51, "eval_tokens": 45, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.65, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is $100 more than the claim made before the event.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.057, "eval_s": 0.99, "tokens_per_s": 45.44, "eval_tokens": 45, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is $100 more than the claim made before the event.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.08, "eval_s": 0.928, "tokens_per_s": 48.47, "eval_tokens": 45, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is $100 more than the claim made before the event.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.067, "eval_s": 1.013, "tokens_per_s": 44.42, "eval_tokens": 45, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.13, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro as announced by Apple was $1,199, which is $100 more than the claim made before the event.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.071, "eval_s": 0.962, "tokens_per_s": 46.8, "eval_tokens": 45, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is $100 more than the claimed $1,199.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.756, "eval_s": 0.96, "tokens_per_s": 52.06, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is $100 more than the claimed $1,199.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.069, "eval_s": 1.033, "tokens_per_s": 48.42, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.15, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is $100 more than the claimed $1,199.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.062, "eval_s": 1.072, "tokens_per_s": 46.65, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is $100 more than the claimed $1,199.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.05, "eval_s": 1.123, "tokens_per_s": 44.54, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "qwen2.5:3b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.), which is $100 more than the claimed $1,199.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.053, "eval_s": 1.124, "tokens_per_s": 44.49, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.22, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "I do not have access to real-time information, including pricing for products that have not yet been released. \n", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.202, "eval_s": 0.482, "tokens_per_s": 51.89, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "I do not have access to real-time information, including pricing for products that have not yet been released. \n", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.208, "eval_s": 0.517, "tokens_per_s": 48.35, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "I do not have access to real-time information, including pricing for products that have not yet been released. \n", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.181, "eval_s": 0.547, "tokens_per_s": 45.68, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.79, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "I do not have access to real-time information, including pricing for products that have not yet been released. \n", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.181, "eval_s": 0.56, "tokens_per_s": 44.61, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "I do not have access to real-time information, including pricing for products that have not yet been released. \n", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.186, "eval_s": 0.475, "tokens_per_s": 52.66, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.69, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 1.667, "eval_s": 0.381, "tokens_per_s": 55.09, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.21, "eval_s": 0.422, "tokens_per_s": 49.73, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.2, "eval_s": 0.472, "tokens_per_s": 44.52, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.73, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.212, "eval_s": 0.41, "tokens_per_s": 51.27, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.67, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.193, "eval_s": 0.491, "tokens_per_s": 42.79, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.72, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299. \n", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 1.736, "eval_s": 0.592, "tokens_per_s": 54.05, "eval_tokens": 32, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299. \n", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.223, "eval_s": 0.695, "tokens_per_s": 46.06, "eval_tokens": 32, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299. \n", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.186, "eval_s": 0.678, "tokens_per_s": 47.22, "eval_tokens": 32, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.91, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299. \n", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.216, "eval_s": 0.618, "tokens_per_s": 51.79, "eval_tokens": 32, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.86, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299. \n", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.19, "eval_s": 0.691, "tokens_per_s": 46.32, "eval_tokens": 32, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.92, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 2.301, "eval_s": 0.387, "tokens_per_s": 54.27, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.72, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.198, "eval_s": 0.513, "tokens_per_s": 40.92, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.74, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.215, "eval_s": 0.405, "tokens_per_s": 51.79, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.21, "eval_s": 0.486, "tokens_per_s": 43.2, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.74, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.219, "eval_s": 0.407, "tokens_per_s": 51.53, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.67, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 2.215, "eval_s": 0.475, "tokens_per_s": 52.65, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.79, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.216, "eval_s": 0.527, "tokens_per_s": 47.48, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.215, "eval_s": 0.486, "tokens_per_s": 51.45, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.211, "eval_s": 0.536, "tokens_per_s": 46.62, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.8, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.192, "eval_s": 0.534, "tokens_per_s": 46.86, "eval_tokens": 25, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.79, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "I do not have access to real-time information, including pricing for products that are not yet released. \n", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.233, "eval_s": 0.489, "tokens_per_s": 49.11, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.8, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "I do not have access to real-time information, including pricing for products that are not yet released. \n", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.192, "eval_s": 0.536, "tokens_per_s": 44.79, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "I do not have access to real-time information, including pricing for products that are not yet released. \n", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.188, "eval_s": 0.532, "tokens_per_s": 45.14, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "I do not have access to real-time information, including pricing for products that are not yet released. \n", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.209, "eval_s": 0.466, "tokens_per_s": 51.46, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.72, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "I do not have access to real-time information, including pricing for products that are not yet released. \n", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.174, "eval_s": 0.535, "tokens_per_s": 44.88, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.74, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 2.065, "eval_s": 0.404, "tokens_per_s": 54.52, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.54, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.221, "eval_s": 0.423, "tokens_per_s": 52.03, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.7, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.201, "eval_s": 0.508, "tokens_per_s": 43.31, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.74, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.222, "eval_s": 0.421, "tokens_per_s": 52.24, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.2, "eval_s": 0.502, "tokens_per_s": 43.82, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.73, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399. \n", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 1.406, "eval_s": 0.621, "tokens_per_s": 53.13, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.13, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399. \n", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.194, "eval_s": 0.703, "tokens_per_s": 46.96, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399. \n", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.193, "eval_s": 0.645, "tokens_per_s": 51.17, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.89, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399. \n", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.202, "eval_s": 0.693, "tokens_per_s": 47.61, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is expected to be between $1,349 and $1,399. \n", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.194, "eval_s": 0.722, "tokens_per_s": 45.72, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 2.642, "eval_s": 0.405, "tokens_per_s": 54.38, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.08, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.191, "eval_s": 0.521, "tokens_per_s": 42.25, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.213, "eval_s": 0.429, "tokens_per_s": 51.25, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.192, "eval_s": 0.536, "tokens_per_s": 41.04, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.206, "eval_s": 0.507, "tokens_per_s": 43.37, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.76, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) according to Apple's press release. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 2.512, "eval_s": 0.632, "tokens_per_s": 53.76, "eval_tokens": 34, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.26, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) according to Apple's press release. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.202, "eval_s": 0.74, "tokens_per_s": 45.97, "eval_tokens": 34, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) according to Apple's press release. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.21, "eval_s": 0.663, "tokens_per_s": 51.29, "eval_tokens": 34, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.92, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) according to Apple's press release. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.214, "eval_s": 0.755, "tokens_per_s": 45.01, "eval_tokens": 34, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 (U.S.) according to Apple's press release. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.188, "eval_s": 0.709, "tokens_per_s": 47.97, "eval_tokens": 34, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "I do not have access to real-time information, including pricing for unreleased products. \n", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.84, "eval_s": 0.363, "tokens_per_s": 55.16, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.31, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "I do not have access to real-time information, including pricing for unreleased products. \n", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.196, "eval_s": 0.465, "tokens_per_s": 43.03, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.7, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "I do not have access to real-time information, including pricing for unreleased products. \n", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.201, "eval_s": 0.466, "tokens_per_s": 42.88, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.71, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "I do not have access to real-time information, including pricing for unreleased products. \n", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.199, "eval_s": 0.428, "tokens_per_s": 46.78, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "I do not have access to real-time information, including pricing for unreleased products. \n", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.196, "eval_s": 0.382, "tokens_per_s": 52.41, "eval_tokens": 20, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.148, "eval_s": 0.416, "tokens_per_s": 52.92, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.62, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.205, "eval_s": 0.507, "tokens_per_s": 43.37, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.223, "eval_s": 0.446, "tokens_per_s": 49.34, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.71, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.209, "eval_s": 0.531, "tokens_per_s": 41.46, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.196, "eval_s": 0.522, "tokens_per_s": 42.13, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "Apple's foldable iPhone Duo is expected to start at $2,099 and go up to $2,299 in the United States. \n", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 1.402, "eval_s": 0.675, "tokens_per_s": 51.82, "eval_tokens": 35, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.15, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "Apple's foldable iPhone Duo is expected to start at $2,099 and go up to $2,299 in the United States. \n", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.201, "eval_s": 0.702, "tokens_per_s": 49.85, "eval_tokens": 35, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "Apple's foldable iPhone Duo is expected to start at $2,099 and go up to $2,299 in the United States. \n", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.184, "eval_s": 0.755, "tokens_per_s": 46.38, "eval_tokens": 35, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.96, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "Apple's foldable iPhone Duo is expected to start at $2,099 and go up to $2,299 in the United States. \n", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.204, "eval_s": 0.797, "tokens_per_s": 43.91, "eval_tokens": 35, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "Apple's foldable iPhone Duo is expected to start at $2,099 and go up to $2,299 in the United States. \n", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.189, "eval_s": 0.754, "tokens_per_s": 46.39, "eval_tokens": 35, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.98, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.328, "eval_s": 0.402, "tokens_per_s": 54.74, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.79, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.214, "eval_s": 0.429, "tokens_per_s": 51.31, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.69, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.2, "eval_s": 0.493, "tokens_per_s": 44.67, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.212, "eval_s": 0.505, "tokens_per_s": 43.56, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.223, "eval_s": 0.433, "tokens_per_s": 50.81, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.7, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.403, "eval_s": 0.403, "tokens_per_s": 54.56, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.89, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.214, "eval_s": 0.454, "tokens_per_s": 48.48, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.72, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.207, "eval_s": 0.507, "tokens_per_s": 43.35, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.76, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.227, "eval_s": 0.425, "tokens_per_s": 51.77, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.68, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "Apple's foldable iPhone Duo starts at $1,999 in the United States. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.19, "eval_s": 0.508, "tokens_per_s": 43.33, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.74, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's starting price is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 2.6, "eval_s": 0.559, "tokens_per_s": 53.67, "eval_tokens": 30, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.24, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's starting price is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.207, "eval_s": 0.579, "tokens_per_s": 51.78, "eval_tokens": 30, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.83, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's starting price is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.214, "eval_s": 0.599, "tokens_per_s": 50.08, "eval_tokens": 30, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.83, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's starting price is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.201, "eval_s": 0.683, "tokens_per_s": 43.9, "eval_tokens": 30, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.93, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's starting price is $1,199 (U.S.). \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.221, "eval_s": 0.582, "tokens_per_s": 51.51, "eval_tokens": 30, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.86, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 2.442, "eval_s": 0.541, "tokens_per_s": 51.73, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.208, "eval_s": 0.671, "tokens_per_s": 41.76, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.93, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.221, "eval_s": 0.548, "tokens_per_s": 51.09, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.223, "eval_s": 0.551, "tokens_per_s": 50.85, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.82, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.222, "eval_s": 0.541, "tokens_per_s": 51.74, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.79, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "VERDICT: YES\nThe iPhone Duo starts at $1,999 (U.S.). \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.332, "eval_s": 0.443, "tokens_per_s": 54.22, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.87, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "VERDICT: YES\nThe iPhone Duo starts at $1,999 (U.S.). \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.211, "eval_s": 0.548, "tokens_per_s": 43.82, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "VERDICT: YES\nThe iPhone Duo starts at $1,999 (U.S.). \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.208, "eval_s": 0.548, "tokens_per_s": 43.8, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "VERDICT: YES\nThe iPhone Duo starts at $1,999 (U.S.). \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.207, "eval_s": 0.557, "tokens_per_s": 43.05, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.83, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "VERDICT: YES\nThe iPhone Duo starts at $1,999 (U.S.). \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.221, "eval_s": 0.571, "tokens_per_s": 42.01, "eval_tokens": 24, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.88, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe starting price of the iPhone Duo is $1,999. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 2.158, "eval_s": 0.423, "tokens_per_s": 54.31, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.66, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe starting price of the iPhone Duo is $1,999. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.238, "eval_s": 0.443, "tokens_per_s": 51.95, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.73, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe starting price of the iPhone Duo is $1,999. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.189, "eval_s": 0.547, "tokens_per_s": 42.01, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe starting price of the iPhone Duo is $1,999. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.206, "eval_s": 0.524, "tokens_per_s": 43.86, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe starting price of the iPhone Duo is $1,999. \n", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.202, "eval_s": 0.492, "tokens_per_s": 46.75, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.75, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's US starting price is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 3.78, "eval_s": 0.554, "tokens_per_s": 48.7, "eval_tokens": 27, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.46, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's US starting price is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.221, "eval_s": 0.53, "tokens_per_s": 50.91, "eval_tokens": 27, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's US starting price is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.231, "eval_s": 0.532, "tokens_per_s": 50.72, "eval_tokens": 27, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.8, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's US starting price is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.236, "eval_s": 0.523, "tokens_per_s": 51.66, "eval_tokens": 27, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro's US starting price is $1,199. \n", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.229, "eval_s": 0.546, "tokens_per_s": 49.43, "eval_tokens": 27, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.82, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "gemma2:2b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 3.984, "eval_s": 0.52, "tokens_per_s": 53.86, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.61, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "gemma2:2b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.212, "eval_s": 0.581, "tokens_per_s": 48.18, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "gemma2:2b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.23, "eval_s": 0.548, "tokens_per_s": 51.14, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "gemma2:2b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.22, "eval_s": 0.553, "tokens_per_s": 50.6, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.84, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "gemma2:2b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe iPhone 18 Pro Max's US starting price is $1,299. \n", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.212, "eval_s": 0.557, "tokens_per_s": 50.23, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.82, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro as it doesn't exist according to my current knowledge.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.09, "eval_s": 1.231, "tokens_per_s": 26.81, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.35, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro as it doesn't exist according to my current knowledge.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.102, "eval_s": 1.22, "tokens_per_s": 27.06, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.36, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro as it doesn't exist according to my current knowledge.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.085, "eval_s": 1.233, "tokens_per_s": 26.76, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.38, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro as it doesn't exist according to my current knowledge.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.076, "eval_s": 1.181, "tokens_per_s": 27.93, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.3, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro as it doesn't exist according to my current knowledge.", "parsed": null, "expected": 1199.0, "correct": false, "ttft_s": 0.098, "eval_s": 1.321, "tokens_per_s": 24.98, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.45, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.831, "eval_s": 0.724, "tokens_per_s": 29.02, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.63, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.102, "eval_s": 0.84, "tokens_per_s": 25.01, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.093, "eval_s": 0.966, "tokens_per_s": 21.73, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.085, "eval_s": 0.806, "tokens_per_s": 26.05, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.92, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-official", "category": "official", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.115, "eval_s": 1.111, "tokens_per_s": 18.9, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.3, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a MacRumors report. The exact price is not specified within the provided information.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.612, "eval_s": 2.022, "tokens_per_s": 24.72, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.78, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a MacRumors report. The exact price is not specified within the provided information.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.076, "eval_s": 2.29, "tokens_per_s": 21.84, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.42, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a MacRumors report. The exact price is not specified within the provided information.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.102, "eval_s": 2.174, "tokens_per_s": 23.0, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.31, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a MacRumors report. The exact price is not specified within the provided information.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.101, "eval_s": 2.133, "tokens_per_s": 23.44, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.28, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is expected to be between $1,249 and $1,299, according to a MacRumors report. The exact price is not specified within the provided information.", "parsed": 1299.0, "expected": 1199.0, "correct": false, "ttft_s": 0.078, "eval_s": 2.298, "tokens_per_s": 21.76, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.42, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.627, "eval_s": 0.838, "tokens_per_s": 25.07, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.51, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.094, "eval_s": 1.046, "tokens_per_s": 20.08, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.2, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.103, "eval_s": 1.078, "tokens_per_s": 19.49, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.22, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.091, "eval_s": 1.134, "tokens_per_s": 18.51, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.27, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 US dollars.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.105, "eval_s": 0.94, "tokens_per_s": 22.35, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 USD, as stated in Apple's press release of 9 September 2026.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.616, "eval_s": 1.763, "tokens_per_s": 22.12, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.6, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 USD, as stated in Apple's press release of 9 September 2026.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.077, "eval_s": 2.017, "tokens_per_s": 19.34, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.16, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 USD, as stated in Apple's press release of 9 September 2026.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.086, "eval_s": 1.807, "tokens_per_s": 21.58, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.93, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 USD, as stated in Apple's press release of 9 September 2026.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.085, "eval_s": 1.836, "tokens_per_s": 21.25, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "pro-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro is $1,199 USD, as stated in Apple's press release of 9 September 2026.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.092, "eval_s": 2.026, "tokens_per_s": 19.25, "eval_tokens": 39, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.16, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro Max. It's possible the model doesn't exist or the name is misspelled.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.381, "eval_s": 1.728, "tokens_per_s": 23.73, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.27, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro Max. It's possible the model doesn't exist or the name is misspelled.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.093, "eval_s": 1.768, "tokens_per_s": 23.19, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.92, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro Max. It's possible the model doesn't exist or the name is misspelled.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.085, "eval_s": 1.827, "tokens_per_s": 22.44, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.96, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro Max. It's possible the model doesn't exist or the name is misspelled.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.095, "eval_s": 1.767, "tokens_per_s": 23.2, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.89, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "I'm sorry, but I don'thy have any information about an iPhone 18 Pro Max. It's possible the model doesn't exist or the name is misspelled.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.09, "eval_s": 1.956, "tokens_per_s": 20.96, "eval_tokens": 41, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.09, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.823, "eval_s": 0.97, "tokens_per_s": 23.72, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.95, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.091, "eval_s": 1.2, "tokens_per_s": 19.16, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.34, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.111, "eval_s": 1.009, "tokens_per_s": 22.8, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.16, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.1, "eval_s": 1.018, "tokens_per_s": 22.59, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.17, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-official", "category": "official", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max in US dollars is $1,299.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.101, "eval_s": 1.054, "tokens_per_s": 21.81, "eval_tokens": 23, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.22, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The information provided does not specify an exact starting price for the iPhone 18 Pro Max; it only gives a range of $1,349 to $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.401, "eval_s": 1.607, "tokens_per_s": 24.89, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.17, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The information provided does not specify an exact starting price for the iPhone 18 Pro Max; it only gives a range of $1,349 to $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.075, "eval_s": 1.784, "tokens_per_s": 22.42, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.92, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The information provided does not specify an exact starting price for the iPhone 18 Pro Max; it only gives a range of $1,349 to $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.105, "eval_s": 2.314, "tokens_per_s": 17.28, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.46, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The information provided does not specify an exact starting price for the iPhone 18 Pro Max; it only gives a range of $1,349 to $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.1, "eval_s": 2.339, "tokens_per_s": 17.1, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.48, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The information provided does not specify an exact starting price for the iPhone 18 Pro Max; it only gives a range of $1,349 to $1,399.", "parsed": 1349.0, "expected": 1299.0, "correct": false, "ttft_s": 0.096, "eval_s": 1.905, "tokens_per_s": 21.0, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.03, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 US dollars.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.632, "eval_s": 0.903, "tokens_per_s": 24.37, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.58, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 US dollars.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.084, "eval_s": 1.016, "tokens_per_s": 21.65, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.13, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 US dollars.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.099, "eval_s": 1.225, "tokens_per_s": 17.96, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.38, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 US dollars.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.107, "eval_s": 1.138, "tokens_per_s": 19.34, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.3, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The starting price of the iPhone 18 Pro Max is $1,299 US dollars.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.1, "eval_s": 1.006, "tokens_per_s": 21.86, "eval_tokens": 22, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.16, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The information provided does not specify the exact starting price of the iPhone 18 Pro Max; it only mentions a range expected by research firm TrendForce. Therefore, the exact starting price in US dollars of the iPhone 18 Pro Max cannot be determined from the given information.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.579, "eval_s": 2.58, "tokens_per_s": 22.86, "eval_tokens": 59, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.36, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The information provided does not specify the exact starting price of the iPhone 18 Pro Max; it only mentions a range expected by research firm TrendForce. Therefore, the exact starting price in US dollars of the iPhone 18 Pro Max cannot be determined from the given information.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.092, "eval_s": 2.731, "tokens_per_s": 21.6, "eval_tokens": 59, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.87, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The information provided does not specify the exact starting price of the iPhone 18 Pro Max; it only mentions a range expected by research firm TrendForce. Therefore, the exact starting price in US dollars of the iPhone 18 Pro Max cannot be determined from the given information.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.09, "eval_s": 2.788, "tokens_per_s": 21.16, "eval_tokens": 59, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.92, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The information provided does not specify the exact starting price of the iPhone 18 Pro Max; it only mentions a range expected by research firm TrendForce. Therefore, the exact starting price in US dollars of the iPhone 18 Pro Max cannot be determined from the given information.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.08, "eval_s": 2.864, "tokens_per_s": 20.6, "eval_tokens": 59, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "promax-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The information provided does not specify the exact starting price of the iPhone 18 Pro Max; it only mentions a range expected by research firm TrendForce. Therefore, the exact starting price in US dollars of the iPhone 18 Pro Max cannot be determined from the given information.", "parsed": null, "expected": 1299.0, "correct": false, "ttft_s": 0.094, "eval_s": 2.9, "tokens_per_s": 20.34, "eval_tokens": 59, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "I'm sorry, but I don'thy have the current pricing information for Apple's foldable iPhone Duo as my training data only goes up until September 2021. I recommend checking the latest information on Apple's official website or a reliable electronics news source.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.644, "eval_s": 2.577, "tokens_per_s": 24.06, "eval_tokens": 62, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.4, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "I'm sorry, but I don'thy have the current pricing information for Apple's foldable iPhone Duo as my training data only goes up until September 2021. I recommend checking the latest information on Apple's official website or a reliable electronics news source.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.101, "eval_s": 2.787, "tokens_per_s": 22.25, "eval_tokens": 62, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.94, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "I'm sorry, but I don'thy have the current pricing information for Apple's foldable iPhone Duo as my training data only goes up until September 2021. I recommend checking the latest information on Apple's official website or a reliable electronics news source.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.069, "eval_s": 2.708, "tokens_per_s": 22.9, "eval_tokens": 62, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "I'm sorry, but I don'thy have the current pricing information for Apple's foldable iPhone Duo as my training data only goes up until September 2021. I recommend checking the latest information on Apple's official website or a reliable electronics news source.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.096, "eval_s": 2.597, "tokens_per_s": 23.87, "eval_tokens": 62, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.74, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-bare", "category": "bare", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "I'm sorry, but I don'thy have the current pricing information for Apple's foldable iPhone Duo as my training data only goes up until September 2021. I recommend checking the latest information on Apple's official website or a reliable electronics news source.", "parsed": null, "expected": 1999.0, "correct": false, "ttft_s": 0.108, "eval_s": 2.726, "tokens_per_s": 22.74, "eval_tokens": 62, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.88, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The foldable iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.789, "eval_s": 0.87, "tokens_per_s": 24.13, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.76, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The foldable iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.098, "eval_s": 0.959, "tokens_per_s": 21.91, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.11, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The foldable iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.08, "eval_s": 0.842, "tokens_per_s": 24.94, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The foldable iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.093, "eval_s": 1.125, "tokens_per_s": 18.66, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.24, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-official", "category": "official", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The foldable iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.09, "eval_s": 0.874, "tokens_per_s": 24.02, "eval_tokens": 21, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.01, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is estimated to be between $2,099 and $2,299, according to a MacRumors report. The exact price within this range is not specified.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.473, "eval_s": 2.241, "tokens_per_s": 24.54, "eval_tokens": 55, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.91, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is estimated to be between $2,099 and $2,299, according to a MacRumors report. The exact price within this range is not specified.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.076, "eval_s": 2.284, "tokens_per_s": 24.08, "eval_tokens": 55, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.41, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is estimated to be between $2,099 and $2,299, according to a MacRumors report. The exact price within this range is not specified.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.086, "eval_s": 2.51, "tokens_per_s": 21.91, "eval_tokens": 55, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.62, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is estimated to be between $2,099 and $2,299, according to a MacRumors report. The exact price within this range is not specified.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.07, "eval_s": 2.411, "tokens_per_s": 22.81, "eval_tokens": 55, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.52, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-rumour", "category": "rumour", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The cost of Apple's foldable iPhone Duo in the United States is estimated to be between $2,099 and $2,299, according to a MacRumors report. The exact price within this range is not specified.", "parsed": 2099.0, "expected": 1999.0, "correct": false, "ttft_s": 0.066, "eval_s": 2.235, "tokens_per_s": 24.61, "eval_tokens": 55, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.35, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.548, "eval_s": 0.661, "tokens_per_s": 27.23, "eval_tokens": 18, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.23, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.086, "eval_s": 0.905, "tokens_per_s": 19.89, "eval_tokens": 18, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.05, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.097, "eval_s": 0.787, "tokens_per_s": 22.87, "eval_tokens": 18, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.93, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.124, "eval_s": 0.792, "tokens_per_s": 22.72, "eval_tokens": 18, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-ro", "category": "both-ro", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The iPhone Duo starts at $1,999 in the United States.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.096, "eval_s": 0.742, "tokens_per_s": 24.24, "eval_tokens": 18, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 2.88, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "The information provided does not specify a single cost for Apple's foldable iPhone Duo; instead, it mentions a starting price of $1,999 and a monthly payment option for 24 months. The MacRumors report suggests an expected price range of $2,099 to $2,299, but this is not confirmed by the given information. Therefore, the exact cost in the United States is not clearly stated.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.549, "eval_s": 3.599, "tokens_per_s": 26.95, "eval_tokens": 97, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 6.38, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "The information provided does not specify a single cost for Apple's foldable iPhone Duo; instead, it mentions a starting price of $1,999 and a monthly payment option for 24 months. The MacRumors report suggests an expected price range of $2,099 to $2,299, but this is not confirmed by the given information. Therefore, the exact cost in the United States is not clearly stated.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.091, "eval_s": 3.697, "tokens_per_s": 26.23, "eval_tokens": 97, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.85, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "The information provided does not specify a single cost for Apple's foldable iPhone Duo; instead, it mentions a starting price of $1,999 and a monthly payment option for 24 months. The MacRumors report suggests an expected price range of $2,099 to $2,299, but this is not confirmed by the given information. Therefore, the exact cost in the United States is not clearly stated.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.107, "eval_s": 3.711, "tokens_per_s": 26.14, "eval_tokens": 97, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.87, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "The information provided does not specify a single cost for Apple's foldable iPhone Duo; instead, it mentions a starting price of $1,999 and a monthly payment option for 24 months. The MacRumors report suggests an expected price range of $2,099 to $2,299, but this is not confirmed by the given information. Therefore, the exact cost in the United States is not clearly stated.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.098, "eval_s": 3.731, "tokens_per_s": 26.0, "eval_tokens": 97, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.86, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "duo-both-or", "category": "both-or", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "The information provided does not specify a single cost for Apple's foldable iPhone Duo; instead, it mentions a starting price of $1,999 and a monthly payment option for 24 months. The MacRumors report suggests an expected price range of $2,099 to $2,299, but this is not confirmed by the given information. Therefore, the exact cost in the United States is not clearly stated.", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.098, "eval_s": 3.812, "tokens_per_s": 25.45, "eval_tokens": 97, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.96, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone 18 Pro announced by Apple was $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.818, "eval_s": 1.277, "tokens_per_s": 25.85, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.27, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone 18 Pro announced by Apple was $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.093, "eval_s": 1.463, "tokens_per_s": 22.56, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.62, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone 18 Pro announced by Apple was $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.093, "eval_s": 1.333, "tokens_per_s": 24.75, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.47, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone 18 Pro announced by Apple was $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.099, "eval_s": 1.274, "tokens_per_s": 25.9, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.44, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone 18 Pro announced by Apple was $1,199 (U.S.).", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.092, "eval_s": 1.606, "tokens_per_s": 20.55, "eval_tokens": 33, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.77, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.) for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.775, "eval_s": 1.722, "tokens_per_s": 24.97, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.64, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.) for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.073, "eval_s": 1.82, "tokens_per_s": 23.62, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.94, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.) for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.099, "eval_s": 1.674, "tokens_per_s": 25.68, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.81, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.) for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.109, "eval_s": 1.899, "tokens_per_s": 22.64, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.07, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price for the iPhone 18 Pro Max announced by Apple was $1,299 (U.S.) for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.105, "eval_s": 1.735, "tokens_per_s": 24.78, "eval_tokens": 43, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.9, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.741, "eval_s": 0.997, "tokens_per_s": 28.09, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.88, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.1, "eval_s": 1.062, "tokens_per_s": 26.35, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.2, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.097, "eval_s": 1.202, "tokens_per_s": 23.3, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.36, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.081, "eval_s": 1.094, "tokens_per_s": 25.59, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.21, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-trendforce-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.106, "eval_s": 1.216, "tokens_per_s": 23.03, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.37, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.745, "eval_s": 1.155, "tokens_per_s": 24.24, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.11, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.077, "eval_s": 1.088, "tokens_per_s": 25.74, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.2, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.095, "eval_s": 1.263, "tokens_per_s": 22.17, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.42, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.114, "eval_s": 1.303, "tokens_per_s": 21.49, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.47, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-fubon-duo", "category": "verdict", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\nThe actual starting price of the iPhone Duo is $1,999 (U.S.).", "parsed": 1999.0, "expected": 1999.0, "correct": true, "ttft_s": 0.096, "eval_s": 1.238, "tokens_per_s": 22.62, "eval_tokens": 28, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.39, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro is $1,199 USD per unit or $49.95 USD per month for a 24-month contract.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.999, "eval_s": 1.985, "tokens_per_s": 25.19, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 5.16, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro is $1,199 USD per unit or $49.95 USD per month for a 24-month contract.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.101, "eval_s": 2.152, "tokens_per_s": 23.23, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.29, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro is $1,199 USD per unit or $49.95 USD per month for a 24-month contract.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.089, "eval_s": 2.375, "tokens_per_s": 21.05, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.55, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro is $1,199 USD per unit or $49.95 USD per month for a 24-month contract.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.072, "eval_s": 2.367, "tokens_per_s": 21.12, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.5, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-pro", "category": "verdict", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro is $1,199 USD per unit or $49.95 USD per month for a 24-month contract.", "parsed": 1199.0, "expected": 1199.0, "correct": true, "ttft_s": 0.091, "eval_s": 2.151, "tokens_per_s": 23.25, "eval_tokens": 50, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.28, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 0, "seed": 42, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro Max is $1,299 USD per month for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.893, "eval_s": 1.799, "tokens_per_s": 22.23, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.97, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 1, "seed": 43, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro Max is $1,299 USD per month for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.067, "eval_s": 1.785, "tokens_per_s": 22.41, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.9, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 2, "seed": 44, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro Max is $1,299 USD per month for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.101, "eval_s": 1.856, "tokens_per_s": 21.55, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 4.0, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 3, "seed": 45, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro Max is $1,299 USD per month for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.095, "eval_s": 1.856, "tokens_per_s": 21.56, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.99, "host_os": "Windows-11-10.0.26200-SP0"}}
{"scenario_id": "verdict-gurman-promax", "category": "verdict", "model": "phi3.5:3.8b", "rep": 4, "seed": 46, "raw_output": "VERDICT: NO\n\nThe actual starting price for the iPhone 18 Pro Max is $1,299 USD per month for a 24-month contract.", "parsed": 1299.0, "expected": 1299.0, "correct": true, "ttft_s": 0.081, "eval_s": 1.734, "tokens_per_s": 23.07, "eval_tokens": 40, "done_reason": "stop", "truncated": false, "error": "", "meta": {"wall_s": 3.86, "host_os": "Windows-11-10.0.26200-SP0"}}