{ "source": "https://modelverdict.ai", "license": "CC BY 4.0", "run_id": "20260928T090853Z-cb9757", "run_date": "2026-09-28", "dataset_version": "0.1.0", "leaderboards": [ { "task": "invoices", "lang": "cs", "level": "all", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 9.4119, "latency_ms_p50": 3608, "latency_ms_p90": 4354, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9995, "doc_accuracy": 0.9933, "usd_per_1000": 4.2591, "latency_ms_p50": 3450, "latency_ms_p90": 5273, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9933, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9867, "usd_per_1000": 0.4962, "latency_ms_p50": 4960, "latency_ms_p90": 7713, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9933, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9933 }, "errors": { "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.999, "doc_accuracy": 0.9867, "usd_per_1000": 10.1173, "latency_ms_p50": 3855, "latency_ms_p90": 5418, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9933, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9933, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 1.0313, "latency_ms_p50": 1802, "latency_ms_p90": 2186, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9533, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 0.9867, "bank_account": 0.9933 }, "errors": { "hallucinated": 7, "wrong_value": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9928, "doc_accuracy": 0.9333, "usd_per_1000": 0.5051, "latency_ms_p50": 3173, "latency_ms_p90": 4223, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9533, "supplier_id": 0.9867, "vat_id": 0.98, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 0.9933, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9933 }, "errors": { "wrong_value": 12, "lost_diacritics": 1, "swapped_dates": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.961, "doc_accuracy": 0.6667, "usd_per_1000": 0.2118, "latency_ms_p50": 10427, "latency_ms_p90": 14675, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 0.9333, "vat_id": 0.9667, "invoice_number": 1, "payment_reference": 0.96, "issue_date": 0.96, "tax_point_date": 0.9533, "due_date": 0.94, "net_by_rate": 0.98, "vat_by_rate": 0.9733, "total": 0.9933, "currency": 1, "bank_account": 0.8667 }, "errors": { "wrong_value": 66, "missing_field": 8, "swapped_dates": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9364, "doc_accuracy": 0.56, "usd_per_1000": 0.4373, "latency_ms_p50": 1999, "latency_ms_p90": 2851, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9133, "supplier_id": 0.9267, "vat_id": 0.94, "invoice_number": 1, "payment_reference": 0.9733, "issue_date": 0.98, "tax_point_date": 0.76, "due_date": 0.9733, "net_by_rate": 0.9, "vat_by_rate": 0.9467, "total": 0.9867, "currency": 0.9733, "bank_account": 0.9 }, "errors": { "wrong_value": 59, "hallucinated": 57, "lost_diacritics": 4, "swapped_dates": 4 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9349, "doc_accuracy": 0.6333, "usd_per_1000": 4.6878, "latency_ms_p50": 2626, "latency_ms_p90": 3553, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9, "supplier_id": 0.8733, "vat_id": 0.8533, "invoice_number": 0.9933, "payment_reference": 0.9533, "issue_date": 0.8867, "tax_point_date": 0.8667, "due_date": 0.9067, "net_by_rate": 0.9933, "vat_by_rate": 0.9867, "total": 1, "currency": 1, "bank_account": 0.94 }, "errors": { "wrong_value": 99, "hallucinated": 18, "missing_field": 6, "lost_diacritics": 2, "swapped_dates": 2 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9179, "doc_accuracy": 0.7667, "usd_per_1000": 1.0185, "latency_ms_p50": 17027, "latency_ms_p90": 29813, "invalid_json_rate": 0.0333, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.94, "vat_id": 0.94, "invoice_number": 0.9467, "payment_reference": 0.94, "issue_date": 0.9467, "tax_point_date": 0.9467, "due_date": 0.9467, "net_by_rate": 0.8, "vat_by_rate": 0.8067, "total": 0.94, "currency": 0.94, "bank_account": 0.9067 }, "errors": { "invalid_json": 65, "missing_field": 41, "wrong_value": 36, "wrong_rate": 16, "hallucinated": 1, "net_vs_total": 1 }, "n_cases": 150, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "cs", "level": "text", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.8743, "latency_ms_p50": 2686, "latency_ms_p90": 3732, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.4951, "latency_ms_p50": 6034, "latency_ms_p90": 8902, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.9135, "latency_ms_p50": 1069, "latency_ms_p90": 1209, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.4604, "latency_ms_p50": 2842, "latency_ms_p90": 4044, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.2113, "latency_ms_p50": 2559, "latency_ms_p90": 3022, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.0152, "latency_ms_p50": 2820, "latency_ms_p90": 4174, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9983, "doc_accuracy": 0.9778, "usd_per_1000": 2.9651, "latency_ms_p50": 1473, "latency_ms_p90": 1683, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "missing_field": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 0.2422, "latency_ms_p50": 9030, "latency_ms_p90": 11492, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "missing_field": 2, "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9675, "doc_accuracy": 0.7556, "usd_per_1000": 0.2551, "latency_ms_p50": 1426, "latency_ms_p90": 1988, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9778, "vat_id": 0.9778, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 0.8222, "due_date": 1, "net_by_rate": 0.9111, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9556 }, "errors": { "hallucinated": 12, "wrong_value": 7 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9197, "doc_accuracy": 0.5333, "usd_per_1000": 0.7421, "latency_ms_p50": 19899, "latency_ms_p90": 33538, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9778, "vat_id": 0.9778, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.5333, "vat_by_rate": 0.5556, "total": 0.9778, "currency": 0.9778, "bank_account": 0.9778 }, "errors": { "wrong_value": 28, "wrong_rate": 16, "missing_field": 2, "net_vs_total": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "cs", "level": "clean", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 13.1965, "latency_ms_p50": 3788, "latency_ms_p90": 4090, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.4873, "latency_ms_p50": 4448, "latency_ms_p90": 7183, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.6141, "latency_ms_p50": 4013, "latency_ms_p90": 5479, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 14.5657, "latency_ms_p50": 3857, "latency_ms_p90": 5248, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9966, "doc_accuracy": 0.9556, "usd_per_1000": 0.7289, "latency_ms_p50": 3625, "latency_ms_p90": 4560, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 0.9778, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1, "swapped_dates": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 1.0712, "latency_ms_p50": 2030, "latency_ms_p90": 2352, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 3 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9538, "doc_accuracy": 0.6222, "usd_per_1000": 0.1974, "latency_ms_p50": 10967, "latency_ms_p90": 15239, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.8889, "vat_id": 0.9556, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 0.9556, "tax_point_date": 0.9556, "due_date": 0.9778, "net_by_rate": 0.9556, "vat_by_rate": 0.9778, "total": 0.9778, "currency": 1, "bank_account": 0.8889 }, "errors": { "wrong_value": 27 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9265, "doc_accuracy": 0.8444, "usd_per_1000": 1.1092, "latency_ms_p50": 15940, "latency_ms_p90": 25468, "invalid_json_rate": 0.0444, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 0.9333, "payment_reference": 0.9333, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.9333, "vat_by_rate": 0.9333, "total": 0.9333, "currency": 0.9333, "bank_account": 0.8444 }, "errors": { "invalid_json": 26, "missing_field": 13, "wrong_value": 4 }, "n_cases": 45, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9231, "doc_accuracy": 0.4444, "usd_per_1000": 0.5117, "latency_ms_p50": 1877, "latency_ms_p90": 2277, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8889, "supplier_id": 0.9111, "vat_id": 0.9333, "invoice_number": 1, "payment_reference": 0.9556, "issue_date": 0.9778, "tax_point_date": 0.6667, "due_date": 1, "net_by_rate": 0.8889, "vat_by_rate": 0.9333, "total": 1, "currency": 0.9778, "bank_account": 0.8667 }, "errors": { "hallucinated": 25, "wrong_value": 18, "lost_diacritics": 2 }, "n_cases": 45, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9214, "doc_accuracy": 0.6, "usd_per_1000": 5.372, "latency_ms_p50": 2486, "latency_ms_p90": 3741, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8889, "supplier_id": 0.8667, "vat_id": 0.8444, "invoice_number": 0.9778, "payment_reference": 0.9333, "issue_date": 0.8667, "tax_point_date": 0.7778, "due_date": 0.8667, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9556 }, "errors": { "wrong_value": 36, "hallucinated": 7, "missing_field": 2, "swapped_dates": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" } ] }, { "task": "invoices", "lang": "cs", "level": "scan", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 10.4868, "latency_ms_p50": 3695, "latency_ms_p90": 4464, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9983, "doc_accuracy": 0.9778, "usd_per_1000": 4.6137, "latency_ms_p50": 3547, "latency_ms_p90": 6085, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9983, "doc_accuracy": 0.9778, "usd_per_1000": 11.8908, "latency_ms_p50": 4233, "latency_ms_p90": 5444, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9778, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9966, "doc_accuracy": 0.9556, "usd_per_1000": 0.4929, "latency_ms_p50": 4783, "latency_ms_p90": 6733, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "wrong_value": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 1.0814, "latency_ms_p50": 1828, "latency_ms_p90": 2186, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 0.9556, "bank_account": 0.9778 }, "errors": { "wrong_value": 3, "hallucinated": 3 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9829, "doc_accuracy": 0.8667, "usd_per_1000": 0.5858, "latency_ms_p50": 3361, "latency_ms_p90": 4223, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9111, "supplier_id": 0.9556, "vat_id": 0.9333, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "wrong_value": 9, "lost_diacritics": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9487, "doc_accuracy": 0.5556, "usd_per_1000": 0.1985, "latency_ms_p50": 11919, "latency_ms_p90": 15914, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9111, "vat_id": 0.9333, "invoice_number": 1, "payment_reference": 0.9556, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9111, "net_by_rate": 1, "vat_by_rate": 0.9556, "total": 1, "currency": 1, "bank_account": 0.8222 }, "errors": { "wrong_value": 25, "missing_field": 4, "swapped_dates": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9214, "doc_accuracy": 0.4667, "usd_per_1000": 0.5124, "latency_ms_p50": 2559, "latency_ms_p90": 2943, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.8889, "vat_id": 0.9111, "invoice_number": 1, "payment_reference": 0.9556, "issue_date": 0.9778, "tax_point_date": 0.7556, "due_date": 0.9556, "net_by_rate": 0.8889, "vat_by_rate": 0.9111, "total": 0.9778, "currency": 0.9333, "bank_account": 0.8889 }, "errors": { "wrong_value": 26, "hallucinated": 16, "swapped_dates": 3, "lost_diacritics": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9094, "doc_accuracy": 0.8889, "usd_per_1000": 1.1724, "latency_ms_p50": 15379, "latency_ms_p90": 32846, "invalid_json_rate": 0.0667, "by_field": { "supplier_name": 0.9111, "supplier_id": 0.9111, "vat_id": 0.9111, "invoice_number": 0.9111, "payment_reference": 0.9111, "issue_date": 0.9111, "tax_point_date": 0.9111, "due_date": 0.9111, "net_by_rate": 0.9111, "vat_by_rate": 0.9111, "total": 0.9111, "currency": 0.9111, "bank_account": 0.8889 }, "errors": { "invalid_json": 39, "missing_field": 13, "wrong_value": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.8906, "doc_accuracy": 0.3556, "usd_per_1000": 5.4072, "latency_ms_p50": 2905, "latency_ms_p90": 3837, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8222, "supplier_id": 0.7111, "vat_id": 0.6667, "invoice_number": 1, "payment_reference": 0.9111, "issue_date": 0.8444, "tax_point_date": 0.8444, "due_date": 0.9333, "net_by_rate": 0.9778, "vat_by_rate": 0.9778, "total": 1, "currency": 1, "bank_account": 0.8889 }, "errors": { "wrong_value": 49, "hallucinated": 11, "missing_field": 3, "lost_diacritics": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" } ] }, { "task": "invoices", "lang": "cs", "level": "photo", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 8.4467, "latency_ms_p50": 3857, "latency_ms_p90": 4384, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.5362, "latency_ms_p50": 5361, "latency_ms_p90": 6488, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.5263, "latency_ms_p50": 3367, "latency_ms_p90": 4783, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 1.1148, "latency_ms_p50": 1821, "latency_ms_p90": 2119, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 9.7578, "latency_ms_p50": 4944, "latency_ms_p90": 6437, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 0.4723, "latency_ms_p50": 3293, "latency_ms_p90": 3794, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8667, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9282, "doc_accuracy": 0.6, "usd_per_1000": 0.5354, "latency_ms_p50": 2851, "latency_ms_p90": 3432, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8667, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9333, "tax_point_date": 0.8667, "due_date": 0.8667, "net_by_rate": 0.9333, "vat_by_rate": 0.9333, "total": 0.9333, "currency": 1, "bank_account": 0.8667 }, "errors": { "wrong_value": 8, "hallucinated": 4, "lost_diacritics": 1, "swapped_dates": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9179, "doc_accuracy": 0.3333, "usd_per_1000": 0.2037, "latency_ms_p50": 10965, "latency_ms_p90": 17598, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9333, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 0.9333, "tax_point_date": 0.8667, "due_date": 0.7333, "net_by_rate": 0.9333, "vat_by_rate": 0.9333, "total": 1, "currency": 1, "bank_account": 0.7333 }, "errors": { "wrong_value": 13, "missing_field": 2, "swapped_dates": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9179, "doc_accuracy": 0.5333, "usd_per_1000": 5.645, "latency_ms_p50": 2995, "latency_ms_p90": 3778, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8667, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.7333, "tax_point_date": 0.8, "due_date": 0.6667, "net_by_rate": 1, "vat_by_rate": 0.9333, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 14, "lost_diacritics": 1, "swapped_dates": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9128, "doc_accuracy": 0.8667, "usd_per_1000": 1.1136, "latency_ms_p50": 15434, "latency_ms_p90": 43358, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8667, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 0.9333, "payment_reference": 0.8667, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.8667, "vat_by_rate": 0.8667, "total": 0.9333, "currency": 0.9333, "bank_account": 0.9333 }, "errors": { "missing_field": 13, "wrong_value": 3, "hallucinated": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "missing_field", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "de", "level": "all", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 1.03, "latency_ms_p50": 1530, "latency_ms_p90": 1909, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9995, "doc_accuracy": 0.9933, "usd_per_1000": 10.652, "latency_ms_p50": 4897, "latency_ms_p90": 6986, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9933 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9954, "doc_accuracy": 0.94, "usd_per_1000": 0.5657, "latency_ms_p50": 6405, "latency_ms_p90": 9388, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9933, "supplier_id": 0.9867, "vat_id": 0.9933, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 0.9933, "due_date": 0.9933, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.98 }, "errors": { "wrong_value": 8, "swapped_dates": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9923, "doc_accuracy": 0.9133, "usd_per_1000": 0.5446, "latency_ms_p50": 3701, "latency_ms_p90": 5093, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9867, "vat_id": 0.9933, "invoice_number": 0.9933, "payment_reference": 1, "issue_date": 0.9867, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.94 }, "errors": { "wrong_value": 13, "missing_field": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 10.4175, "latency_ms_p50": 3751, "latency_ms_p90": 5762, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9933, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.8933, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.98 }, "errors": { "missing_field": 17, "wrong_value": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9738, "doc_accuracy": 0.7467, "usd_per_1000": 4.7201, "latency_ms_p50": 2478, "latency_ms_p90": 3177, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9733, "supplier_id": 0.9867, "vat_id": 0.96, "invoice_number": 1, "payment_reference": 0.9267, "issue_date": 0.98, "tax_point_date": 0.9067, "due_date": 0.9733, "net_by_rate": 0.98, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9733 }, "errors": { "hallucinated": 26, "wrong_value": 18, "missing_field": 5, "swapped_dates": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9723, "doc_accuracy": 0.96, "usd_per_1000": 7.1537, "latency_ms_p50": 4732, "latency_ms_p90": 8147, "invalid_json_rate": 0.0267, "by_field": { "supplier_name": 0.9733, "supplier_id": 0.9733, "vat_id": 0.9667, "invoice_number": 0.9733, "payment_reference": 0.9733, "issue_date": 0.9733, "tax_point_date": 0.9733, "due_date": 0.9733, "net_by_rate": 0.9667, "vat_by_rate": 0.9733, "total": 0.9733, "currency": 0.9733, "bank_account": 0.9733 }, "errors": { "invalid_json": 52, "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9487, "doc_accuracy": 0.64, "usd_per_1000": 0.199, "latency_ms_p50": 7704, "latency_ms_p90": 13269, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9867, "supplier_id": 0.9933, "vat_id": 0.94, "invoice_number": 0.9933, "payment_reference": 0.9867, "issue_date": 0.8733, "tax_point_date": 0.92, "due_date": 0.9067, "net_by_rate": 0.9467, "vat_by_rate": 0.9733, "total": 0.9933, "currency": 0.9933, "bank_account": 0.8267 }, "errors": { "wrong_value": 90, "missing_field": 7, "swapped_dates": 3 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9462, "doc_accuracy": 0.5, "usd_per_1000": 0.4365, "latency_ms_p50": 1960, "latency_ms_p90": 2664, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9857, "supplier_id": 0.8, "vat_id": 0.9286, "invoice_number": 1, "payment_reference": 0.8714, "issue_date": 1, "tax_point_date": 0.8786, "due_date": 0.9857, "net_by_rate": 0.9429, "vat_by_rate": 0.9929, "total": 1, "currency": 1, "bank_account": 0.9143 }, "errors": { "hallucinated": 64, "wrong_value": 22, "missing_field": 10, "swapped_dates": 2 }, "n_cases": 140, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9349, "doc_accuracy": 0.8733, "usd_per_1000": 1.1188, "latency_ms_p50": 19027, "latency_ms_p90": 33896, "invalid_json_rate": 0.06, "by_field": { "supplier_name": 0.94, "supplier_id": 0.94, "vat_id": 0.94, "invoice_number": 0.9267, "payment_reference": 0.8867, "issue_date": 0.94, "tax_point_date": 0.94, "due_date": 0.94, "net_by_rate": 0.94, "vat_by_rate": 0.94, "total": 0.94, "currency": 0.94, "bank_account": 0.94 }, "errors": { "invalid_json": 117, "hallucinated": 8, "wrong_value": 2 }, "n_cases": 150, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "de", "level": "text", "bar": 0.95, "rows": [ { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.492, "latency_ms_p50": 6669, "latency_ms_p90": 9667, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.9111, "latency_ms_p50": 1100, "latency_ms_p90": 1283, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.2175, "latency_ms_p50": 2983, "latency_ms_p90": 3682, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.8529, "latency_ms_p50": 3109, "latency_ms_p90": 4157, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.9868, "latency_ms_p50": 27816, "latency_ms_p90": 50076, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 0.227, "latency_ms_p50": 7019, "latency_ms_p90": 12184, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9333, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 3 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 2.9567, "latency_ms_p50": 1392, "latency_ms_p90": 1702, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9778, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9556, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 2, "hallucinated": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9932, "doc_accuracy": 0.9111, "usd_per_1000": 5.8922, "latency_ms_p50": 3272, "latency_ms_p90": 5883, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9778, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9556, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "missing_field": 3, "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.978, "doc_accuracy": 0.7857, "usd_per_1000": 0.2471, "latency_ms_p50": 1326, "latency_ms_p90": 1709, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9762, "invoice_number": 1, "payment_reference": 0.9048, "issue_date": 1, "tax_point_date": 0.9762, "due_date": 1, "net_by_rate": 0.8571, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 10, "missing_field": 1, "wrong_value": 1 }, "n_cases": 42, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9778, "doc_accuracy": 0.9778, "usd_per_1000": 6.2407, "latency_ms_p50": 3872, "latency_ms_p90": 7363, "invalid_json_rate": 0.0222, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9778, "vat_id": 0.9778, "invoice_number": 0.9778, "payment_reference": 0.9778, "issue_date": 0.9778, "tax_point_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.9778, "vat_by_rate": 0.9778, "total": 0.9778, "currency": 0.9778, "bank_account": 0.9778 }, "errors": { "invalid_json": 13 }, "n_cases": 45, "passes_bar": true, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" } ] }, { "task": "invoices", "lang": "de", "level": "clean", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 1.0733, "latency_ms_p50": 1727, "latency_ms_p90": 1932, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 15.2566, "latency_ms_p50": 5582, "latency_ms_p90": 8432, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9966, "doc_accuracy": 0.9556, "usd_per_1000": 0.5534, "latency_ms_p50": 5725, "latency_ms_p90": 9363, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9778, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9966, "doc_accuracy": 0.9556, "usd_per_1000": 0.7681, "latency_ms_p50": 4092, "latency_ms_p90": 5171, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9556 }, "errors": { "wrong_value": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 13.2731, "latency_ms_p50": 3761, "latency_ms_p90": 4940, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.8889, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "missing_field": 5, "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9658, "doc_accuracy": 0.6889, "usd_per_1000": 5.4305, "latency_ms_p50": 2503, "latency_ms_p90": 3311, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 1, "vat_id": 0.9111, "invoice_number": 1, "payment_reference": 0.8889, "issue_date": 0.9778, "tax_point_date": 0.8667, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "hallucinated": 11, "wrong_value": 8, "missing_field": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9521, "doc_accuracy": 0.9111, "usd_per_1000": 8.8069, "latency_ms_p50": 5108, "latency_ms_p90": 8285, "invalid_json_rate": 0.0444, "by_field": { "supplier_name": 0.9556, "supplier_id": 0.9556, "vat_id": 0.9333, "invoice_number": 0.9556, "payment_reference": 0.9556, "issue_date": 0.9556, "tax_point_date": 0.9556, "due_date": 0.9556, "net_by_rate": 0.9333, "vat_by_rate": 0.9556, "total": 0.9556, "currency": 0.9556, "bank_account": 0.9556 }, "errors": { "invalid_json": 26, "wrong_value": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9385, "doc_accuracy": 0.5111, "usd_per_1000": 0.1859, "latency_ms_p50": 7433, "latency_ms_p90": 11387, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 1, "vat_id": 0.9556, "invoice_number": 0.9778, "payment_reference": 0.9556, "issue_date": 0.8444, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.9111, "vat_by_rate": 0.9556, "total": 0.9778, "currency": 1, "bank_account": 0.7778 }, "errors": { "wrong_value": 35, "missing_field": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9353, "doc_accuracy": 0.3636, "usd_per_1000": 0.5141, "latency_ms_p50": 1806, "latency_ms_p90": 2273, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.7045, "vat_id": 0.8864, "invoice_number": 1, "payment_reference": 0.8636, "issue_date": 1, "tax_point_date": 0.7955, "due_date": 0.9773, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9318 }, "errors": { "hallucinated": 26, "wrong_value": 6, "missing_field": 5 }, "n_cases": 44, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8838, "doc_accuracy": 0.8222, "usd_per_1000": 1.2649, "latency_ms_p50": 16696, "latency_ms_p90": 26372, "invalid_json_rate": 0.1111, "by_field": { "supplier_name": 0.8889, "supplier_id": 0.8889, "vat_id": 0.8889, "invoice_number": 0.8667, "payment_reference": 0.8444, "issue_date": 0.8889, "tax_point_date": 0.8889, "due_date": 0.8889, "net_by_rate": 0.8889, "vat_by_rate": 0.8889, "total": 0.8889, "currency": 0.8889, "bank_account": 0.8889 }, "errors": { "invalid_json": 65, "hallucinated": 2, "wrong_value": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "de", "level": "scan", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 1.0799, "latency_ms_p50": 1606, "latency_ms_p90": 1929, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9983, "doc_accuracy": 0.9778, "usd_per_1000": 13.1538, "latency_ms_p50": 5077, "latency_ms_p90": 7039, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 0.6076, "latency_ms_p50": 7201, "latency_ms_p90": 8817, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9778, "vat_id": 0.9778, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 0.9778, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 6 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9863, "doc_accuracy": 0.8222, "usd_per_1000": 11.8759, "latency_ms_p50": 3838, "latency_ms_p90": 5963, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.8222, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 8 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9829, "doc_accuracy": 0.8222, "usd_per_1000": 0.6634, "latency_ms_p50": 4068, "latency_ms_p90": 5384, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9556, "vat_id": 0.9778, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9556, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8889 }, "errors": { "wrong_value": 8, "missing_field": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9778, "doc_accuracy": 0.9778, "usd_per_1000": 6.6488, "latency_ms_p50": 4988, "latency_ms_p90": 9991, "invalid_json_rate": 0.0222, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9778, "vat_id": 0.9778, "invoice_number": 0.9778, "payment_reference": 0.9778, "issue_date": 0.9778, "tax_point_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.9778, "vat_by_rate": 0.9778, "total": 0.9778, "currency": 0.9778, "bank_account": 0.9778 }, "errors": { "invalid_json": 13 }, "n_cases": 45, "passes_bar": true, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9692, "doc_accuracy": 0.6667, "usd_per_1000": 5.4545, "latency_ms_p50": 2767, "latency_ms_p90": 3225, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9556, "vat_id": 0.9778, "invoice_number": 1, "payment_reference": 0.8889, "issue_date": 0.9778, "tax_point_date": 0.9111, "due_date": 0.9556, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "hallucinated": 11, "wrong_value": 7 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9692, "doc_accuracy": 0.8667, "usd_per_1000": 1.1487, "latency_ms_p50": 16648, "latency_ms_p90": 24926, "invalid_json_rate": 0.0222, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9778, "vat_id": 0.9778, "invoice_number": 0.9778, "payment_reference": 0.8667, "issue_date": 0.9778, "tax_point_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.9778, "vat_by_rate": 0.9778, "total": 0.9778, "currency": 0.9778, "bank_account": 0.9778 }, "errors": { "invalid_json": 13, "hallucinated": 5 }, "n_cases": 45, "passes_bar": true, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9287, "doc_accuracy": 0.3659, "usd_per_1000": 0.5161, "latency_ms_p50": 2364, "latency_ms_p90": 2794, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9756, "supplier_id": 0.7073, "vat_id": 0.9024, "invoice_number": 1, "payment_reference": 0.8293, "issue_date": 1, "tax_point_date": 0.9024, "due_date": 1, "net_by_rate": 0.9512, "vat_by_rate": 0.9756, "total": 1, "currency": 1, "bank_account": 0.8293 }, "errors": { "hallucinated": 22, "wrong_value": 11, "missing_field": 4, "swapped_dates": 1 }, "n_cases": 41, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9179, "doc_accuracy": 0.5111, "usd_per_1000": 0.1872, "latency_ms_p50": 8651, "latency_ms_p90": 13315, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9778, "vat_id": 0.9556, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.7778, "tax_point_date": 0.8444, "due_date": 0.8, "net_by_rate": 0.9111, "vat_by_rate": 0.9556, "total": 1, "currency": 0.9778, "bank_account": 0.7556 }, "errors": { "wrong_value": 42, "swapped_dates": 3, "missing_field": 3 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" } ] }, { "task": "invoices", "lang": "de", "level": "photo", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 1.1068, "latency_ms_p50": 1609, "latency_ms_p90": 1823, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 6.448, "latency_ms_p50": 4808, "latency_ms_p90": 6024, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 9.7305, "latency_ms_p50": 5428, "latency_ms_p90": 6576, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9949, "doc_accuracy": 0.9333, "usd_per_1000": 0.6975, "latency_ms_p50": 6384, "latency_ms_p90": 10996, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 0.9333, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "swapped_dates": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "swapped_dates", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 11.0515, "latency_ms_p50": 3820, "latency_ms_p90": 15338, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9333, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 1, "missing_field": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9846, "doc_accuracy": 0.8, "usd_per_1000": 0.499, "latency_ms_p50": 3546, "latency_ms_p90": 5097, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 0.9333, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8667 }, "errors": { "wrong_value": 3 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9487, "doc_accuracy": 0.6, "usd_per_1000": 5.6755, "latency_ms_p50": 2736, "latency_ms_p90": 3525, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 0.9333, "tax_point_date": 0.7333, "due_date": 0.8667, "net_by_rate": 0.9333, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 3, "hallucinated": 3, "swapped_dates": 2, "missing_field": 2 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9349, "doc_accuracy": 0.4615, "usd_per_1000": 0.5349, "latency_ms_p50": 2495, "latency_ms_p90": 2983, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9231, "supplier_id": 0.7692, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9231, "issue_date": 1, "tax_point_date": 0.7692, "due_date": 0.9231, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8462 }, "errors": { "hallucinated": 6, "wrong_value": 4, "swapped_dates": 1 }, "n_cases": 13, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9333, "doc_accuracy": 0.5333, "usd_per_1000": 0.1899, "latency_ms_p50": 9081, "latency_ms_p90": 13354, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.8667, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.8667, "tax_point_date": 0.8667, "due_date": 0.8667, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.6667 }, "errors": { "wrong_value": 13 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.7897, "doc_accuracy": 0.6667, "usd_per_1000": 0.987, "latency_ms_p50": 15594, "latency_ms_p90": 23333, "invalid_json_rate": 0.2, "by_field": { "supplier_name": 0.8, "supplier_id": 0.8, "vat_id": 0.8, "invoice_number": 0.7333, "payment_reference": 0.7333, "issue_date": 0.8, "tax_point_date": 0.8, "due_date": 0.8, "net_by_rate": 0.8, "vat_by_rate": 0.8, "total": 0.8, "currency": 0.8, "bank_account": 0.8 }, "errors": { "invalid_json": 39, "hallucinated": 1, "wrong_value": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "en", "level": "all", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9994, "doc_accuracy": 0.9933, "usd_per_1000": 9.6483, "latency_ms_p50": 4074, "latency_ms_p90": 5519, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9933 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9982, "doc_accuracy": 0.9867, "usd_per_1000": 8.8374, "latency_ms_p50": 3160, "latency_ms_p90": 3631, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.9867, "vat_by_rate": 0.9933, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 2, "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9982, "doc_accuracy": 0.98, "usd_per_1000": 3.8361, "latency_ms_p50": 3225, "latency_ms_p90": 4555, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.98 }, "errors": { "wrong_value": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.997, "doc_accuracy": 0.9667, "usd_per_1000": 0.5074, "latency_ms_p50": 3435, "latency_ms_p90": 4678, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 0.9933, "payment_reference": 0.98, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9933 }, "errors": { "hallucinated": 3, "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9939, "doc_accuracy": 0.9333, "usd_per_1000": 0.5188, "latency_ms_p50": 5737, "latency_ms_p90": 9427, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 0.9933, "bank_account": 0.9733 }, "errors": { "wrong_value": 5, "hallucinated": 5 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9921, "doc_accuracy": 0.92, "usd_per_1000": 0.8663, "latency_ms_p50": 1700, "latency_ms_p90": 2034, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 0.9733, "due_date": 0.9933, "net_by_rate": 0.98, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 8, "hallucinated": 5 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9776, "doc_accuracy": 0.7933, "usd_per_1000": 4.3725, "latency_ms_p50": 2316, "latency_ms_p90": 5374, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9467, "issue_date": 0.9867, "due_date": 0.9933, "net_by_rate": 0.88, "vat_by_rate": 0.9867, "total": 1, "currency": 1, "bank_account": 0.96 }, "errors": { "wrong_value": 28, "hallucinated": 8, "wrong_rate": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9764, "doc_accuracy": 0.8133, "usd_per_1000": 0.1675, "latency_ms_p50": 8869, "latency_ms_p90": 11901, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9933, "supplier_id": 0.98, "invoice_number": 1, "payment_reference": 0.9933, "issue_date": 0.9867, "due_date": 0.9867, "net_by_rate": 0.9533, "vat_by_rate": 0.9867, "total": 1, "currency": 1, "bank_account": 0.86 }, "errors": { "wrong_value": 38, "wrong_rate": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9558, "doc_accuracy": 0.58, "usd_per_1000": 0.4076, "latency_ms_p50": 1802, "latency_ms_p90": 2647, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9933, "invoice_number": 1, "payment_reference": 0.8733, "issue_date": 0.9733, "due_date": 0.9867, "net_by_rate": 0.72, "vat_by_rate": 0.9933, "total": 1, "currency": 0.9933, "bank_account": 0.98 }, "errors": { "wrong_rate": 39, "wrong_value": 20, "hallucinated": 13, "swapped_dates": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_rate", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8236, "doc_accuracy": 0.5733, "usd_per_1000": 1.1543, "latency_ms_p50": 17427, "latency_ms_p90": 37173, "invalid_json_rate": 0.0933, "by_field": { "supplier_name": 0.8533, "supplier_id": 0.8867, "invoice_number": 0.8533, "payment_reference": 0.7933, "issue_date": 0.8533, "due_date": 0.8533, "net_by_rate": 0.7, "vat_by_rate": 0.76, "total": 0.8533, "currency": 0.84, "bank_account": 0.8133 }, "errors": { "invalid_json": 154, "missing_field": 83, "wrong_rate": 23, "wrong_value": 18, "hallucinated": 13 }, "n_cases": 150, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "en", "level": "text", "bar": 0.95, "rows": [ { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.5161, "latency_ms_p50": 7386, "latency_ms_p90": 11591, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 2.9193, "latency_ms_p50": 2292, "latency_ms_p90": 3558, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.1812, "latency_ms_p50": 7413, "latency_ms_p90": 10416, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.1805, "latency_ms_p50": 2651, "latency_ms_p90": 3772, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.2831, "latency_ms_p50": 3234, "latency_ms_p90": 5149, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.998, "doc_accuracy": 0.9778, "usd_per_1000": 3.9142, "latency_ms_p50": 2368, "latency_ms_p90": 3335, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.9778, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.998, "doc_accuracy": 0.9778, "usd_per_1000": 2.2418, "latency_ms_p50": 1173, "latency_ms_p90": 1533, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.9778, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9939, "doc_accuracy": 0.9556, "usd_per_1000": 0.718, "latency_ms_p50": 962, "latency_ms_p90": 1043, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9556, "due_date": 0.9778, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 3 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9838, "doc_accuracy": 0.8444, "usd_per_1000": 0.1901, "latency_ms_p50": 1187, "latency_ms_p90": 1557, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.8889, "issue_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.9778, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 4, "hallucinated": 3, "wrong_rate": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9172, "doc_accuracy": 0.4889, "usd_per_1000": 0.9702, "latency_ms_p50": 18679, "latency_ms_p90": 42527, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 1, "due_date": 1, "net_by_rate": 0.5111, "vat_by_rate": 0.6667, "total": 1, "currency": 0.9778, "bank_account": 1 }, "errors": { "wrong_rate": 19, "wrong_value": 9, "missing_field": 7, "hallucinated": 6 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_rate", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "en", "level": "clean", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 12.19, "latency_ms_p50": 3267, "latency_ms_p90": 3631, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 13.8084, "latency_ms_p50": 4193, "latency_ms_p90": 5907, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.996, "doc_accuracy": 0.9556, "usd_per_1000": 0.9272, "latency_ms_p50": 1824, "latency_ms_p90": 2152, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9778, "issue_date": 0.9778, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1, "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.996, "doc_accuracy": 0.9556, "usd_per_1000": 4.1338, "latency_ms_p50": 3533, "latency_ms_p90": 4775, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9556 }, "errors": { "wrong_value": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.996, "doc_accuracy": 0.9556, "usd_per_1000": 0.7251, "latency_ms_p50": 3796, "latency_ms_p90": 5315, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9556, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9919, "doc_accuracy": 0.9111, "usd_per_1000": 0.4957, "latency_ms_p50": 4555, "latency_ms_p90": 8879, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "hallucinated": 3, "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9818, "doc_accuracy": 0.8444, "usd_per_1000": 0.1605, "latency_ms_p50": 9201, "latency_ms_p90": 12274, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.9778, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8667 }, "errors": { "wrong_value": 9 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9798, "doc_accuracy": 0.8222, "usd_per_1000": 5.248, "latency_ms_p50": 2282, "latency_ms_p90": 6512, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 0.9778, "due_date": 1, "net_by_rate": 0.9333, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 6, "hallucinated": 3, "wrong_rate": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9556, "doc_accuracy": 0.5556, "usd_per_1000": 0.4985, "latency_ms_p50": 1736, "latency_ms_p90": 2116, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9778, "invoice_number": 1, "payment_reference": 0.8889, "issue_date": 0.9778, "due_date": 1, "net_by_rate": 0.6667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_rate": 15, "hallucinated": 4, "wrong_value": 3 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_rate", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.7717, "doc_accuracy": 0.6, "usd_per_1000": 1.1297, "latency_ms_p50": 15210, "latency_ms_p90": 28007, "invalid_json_rate": 0.1333, "by_field": { "supplier_name": 0.7778, "supplier_id": 0.8444, "invoice_number": 0.7778, "payment_reference": 0.7333, "issue_date": 0.7778, "due_date": 0.7778, "net_by_rate": 0.7778, "vat_by_rate": 0.7778, "total": 0.7778, "currency": 0.7778, "bank_account": 0.6889 }, "errors": { "invalid_json": 66, "missing_field": 38, "wrong_value": 5, "hallucinated": 2, "wrong_rate": 2 }, "n_cases": 45, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "en", "level": "scan", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 10.6568, "latency_ms_p50": 3319, "latency_ms_p90": 3696, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 12.0443, "latency_ms_p50": 4313, "latency_ms_p90": 5469, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 45, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.998, "doc_accuracy": 0.9778, "usd_per_1000": 4.3571, "latency_ms_p50": 3241, "latency_ms_p90": 4550, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9778 }, "errors": { "wrong_value": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.996, "doc_accuracy": 0.9556, "usd_per_1000": 0.6262, "latency_ms_p50": 3740, "latency_ms_p90": 4870, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 0.9778, "payment_reference": 0.9778, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1, "hallucinated": 1 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9899, "doc_accuracy": 0.8889, "usd_per_1000": 0.5453, "latency_ms_p50": 5351, "latency_ms_p90": 8515, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9556, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 0.9778, "bank_account": 0.9556 }, "errors": { "wrong_value": 3, "hallucinated": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9879, "doc_accuracy": 0.8667, "usd_per_1000": 0.9301, "latency_ms_p50": 1791, "latency_ms_p90": 2032, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9111, "issue_date": 0.9778, "due_date": 1, "net_by_rate": 0.9778, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 4, "wrong_value": 2 }, "n_cases": 45, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9657, "doc_accuracy": 0.7333, "usd_per_1000": 0.162, "latency_ms_p50": 9798, "latency_ms_p90": 11657, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9778, "supplier_id": 0.9556, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.9556, "vat_by_rate": 0.9556, "total": 1, "currency": 1, "bank_account": 0.7778 }, "errors": { "wrong_value": 17 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9596, "doc_accuracy": 0.6222, "usd_per_1000": 5.2725, "latency_ms_p50": 2837, "latency_ms_p90": 5374, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.8889, "issue_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.8, "vat_by_rate": 0.9778, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 15, "hallucinated": 5 }, "n_cases": 45, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9354, "doc_accuracy": 0.3778, "usd_per_1000": 0.4992, "latency_ms_p50": 2335, "latency_ms_p90": 2788, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.8222, "issue_date": 0.9778, "due_date": 0.9778, "net_by_rate": 0.5556, "vat_by_rate": 1, "total": 1, "currency": 0.9778, "bank_account": 0.9778 }, "errors": { "wrong_rate": 18, "wrong_value": 7, "hallucinated": 6, "swapped_dates": 1 }, "n_cases": 45, "passes_bar": false, "top_error": "wrong_rate", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8141, "doc_accuracy": 0.6, "usd_per_1000": 1.3487, "latency_ms_p50": 17435, "latency_ms_p90": 36557, "invalid_json_rate": 0.0889, "by_field": { "supplier_name": 0.8222, "supplier_id": 0.8667, "invoice_number": 0.8222, "payment_reference": 0.7333, "issue_date": 0.8222, "due_date": 0.8222, "net_by_rate": 0.8, "vat_by_rate": 0.8444, "total": 0.8222, "currency": 0.8, "bank_account": 0.8 }, "errors": { "invalid_json": 44, "missing_field": 38, "hallucinated": 5, "wrong_value": 3, "wrong_rate": 2 }, "n_cases": 45, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "en", "level": "photo", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.1305, "latency_ms_p50": 3362, "latency_ms_p90": 5045, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 15, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9939, "doc_accuracy": 0.9333, "usd_per_1000": 0.5169, "latency_ms_p50": 5338, "latency_ms_p90": 7755, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9939, "doc_accuracy": 0.9333, "usd_per_1000": 0.4784, "latency_ms_p50": 3709, "latency_ms_p90": 4204, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9939, "doc_accuracy": 0.9333, "usd_per_1000": 9.0758, "latency_ms_p50": 4856, "latency_ms_p90": 5873, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9333 }, "errors": { "wrong_value": 1 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9879, "doc_accuracy": 0.9333, "usd_per_1000": 8.0913, "latency_ms_p50": 3176, "latency_ms_p90": 3589, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.9333, "vat_by_rate": 0.9333, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9879, "doc_accuracy": 0.8667, "usd_per_1000": 0.9374, "latency_ms_p50": 1700, "latency_ms_p90": 2090, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.8667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9636, "doc_accuracy": 0.6667, "usd_per_1000": 5.4385, "latency_ms_p50": 2815, "latency_ms_p90": 7530, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "due_date": 1, "net_by_rate": 0.6667, "vat_by_rate": 0.9333, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 6 }, "n_cases": 15, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9333, "doc_accuracy": 0.4667, "usd_per_1000": 0.5128, "latency_ms_p50": 2597, "latency_ms_p90": 3148, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 0.9333, "due_date": 1, "net_by_rate": 0.6, "vat_by_rate": 0.9333, "total": 1, "currency": 1, "bank_account": 0.8667 }, "errors": { "wrong_value": 6, "wrong_rate": 5 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9212, "doc_accuracy": 0.4, "usd_per_1000": 0.1644, "latency_ms_p50": 10362, "latency_ms_p90": 11591, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9333, "invoice_number": 1, "payment_reference": 0.9333, "issue_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.7333, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.6667 }, "errors": { "wrong_value": 12, "wrong_rate": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.7273, "doc_accuracy": 0.6667, "usd_per_1000": 1.1969, "latency_ms_p50": 19570, "latency_ms_p90": 28157, "invalid_json_rate": 0.2667, "by_field": { "supplier_name": 0.7333, "supplier_id": 0.7333, "invoice_number": 0.7333, "payment_reference": 0.7333, "issue_date": 0.7333, "due_date": 0.7333, "net_by_rate": 0.7333, "vat_by_rate": 0.7333, "total": 0.7333, "currency": 0.7333, "bank_account": 0.6667 }, "errors": { "invalid_json": 44, "wrong_value": 1 }, "n_cases": 15, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "es", "level": "all", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9915, "doc_accuracy": 0.89, "usd_per_1000": 10.7592, "latency_ms_p50": 5692, "latency_ms_p90": 7507, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.98, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.92, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.99 }, "errors": { "missing_field": 8, "wrong_value": 3 }, "n_cases": 100, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9908, "doc_accuracy": 0.88, "usd_per_1000": 0.5521, "latency_ms_p50": 3902, "latency_ms_p90": 5214, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.99, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.98, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.91 }, "errors": { "wrong_value": 10, "hallucinated": 2 }, "n_cases": 100, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9825, "doc_accuracy": 0.7722, "usd_per_1000": 0.7056, "latency_ms_p50": 5820, "latency_ms_p90": 13530, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9367, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9747, "issue_date": 1, "tax_point_date": 0.9873, "due_date": 1, "net_by_rate": 0.8861, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9873 }, "errors": { "missing_field": 9, "hallucinated": 3, "lost_diacritics": 3, "wrong_value": 3 }, "n_cases": 79, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9823, "doc_accuracy": 0.79, "usd_per_1000": 1.0491, "latency_ms_p50": 1569, "latency_ms_p90": 2142, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.98, "supplier_id": 1, "vat_id": 0.83, "invoice_number": 1, "payment_reference": 0.99, "issue_date": 0.99, "tax_point_date": 0.98, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 19, "wrong_value": 4 }, "n_cases": 100, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.97, "doc_accuracy": 0.67, "usd_per_1000": 4.7463, "latency_ms_p50": 2487, "latency_ms_p90": 3994, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.99, "supplier_id": 1, "vat_id": 0.94, "invoice_number": 1, "payment_reference": 0.93, "issue_date": 1, "tax_point_date": 0.93, "due_date": 0.97, "net_by_rate": 0.85, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 20, "missing_field": 18, "wrong_value": 1 }, "n_cases": 100, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9592, "doc_accuracy": 0.95, "usd_per_1000": 7.8637, "latency_ms_p50": 5272, "latency_ms_p90": 9979, "invalid_json_rate": 0.04, "by_field": { "supplier_name": 0.95, "supplier_id": 0.96, "vat_id": 0.96, "invoice_number": 0.96, "payment_reference": 0.96, "issue_date": 0.96, "tax_point_date": 0.96, "due_date": 0.96, "net_by_rate": 0.96, "vat_by_rate": 0.96, "total": 0.96, "currency": 0.96, "bank_account": 0.96 }, "errors": { "invalid_json": 52, "wrong_value": 1 }, "n_cases": 100, "passes_bar": true, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.954, "doc_accuracy": 0.5402, "usd_per_1000": 0.2039, "latency_ms_p50": 8667, "latency_ms_p90": 11107, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.954, "supplier_id": 0.931, "vat_id": 0.9885, "invoice_number": 1, "payment_reference": 0.954, "issue_date": 0.9885, "tax_point_date": 1, "due_date": 0.977, "net_by_rate": 0.8161, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.7931 }, "errors": { "wrong_value": 35, "missing_field": 16, "hallucinated": 1 }, "n_cases": 87, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9362, "doc_accuracy": 0.77, "usd_per_1000": 11.8874, "latency_ms_p50": 4045, "latency_ms_p90": 6706, "invalid_json_rate": 0.05, "by_field": { "supplier_name": 0.94, "supplier_id": 0.95, "vat_id": 0.95, "invoice_number": 0.95, "payment_reference": 0.94, "issue_date": 0.95, "tax_point_date": 0.95, "due_date": 0.95, "net_by_rate": 0.79, "vat_by_rate": 0.95, "total": 0.95, "currency": 0.95, "bank_account": 0.95 }, "errors": { "invalid_json": 65, "missing_field": 16, "hallucinated": 1, "wrong_value": 1 }, "n_cases": 100, "passes_bar": false, "top_error": "invalid_json", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9194, "doc_accuracy": 0.3095, "usd_per_1000": 0.4445, "latency_ms_p50": 2150, "latency_ms_p90": 2931, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9762, "supplier_id": 0.9881, "vat_id": 0.5357, "invoice_number": 1, "payment_reference": 0.881, "issue_date": 1, "tax_point_date": 0.6905, "due_date": 1, "net_by_rate": 0.9643, "vat_by_rate": 0.9762, "total": 0.9881, "currency": 1, "bank_account": 0.9524 }, "errors": { "hallucinated": 72, "wrong_value": 12, "swapped_dates": 3, "missing_field": 1 }, "n_cases": 84, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8792, "doc_accuracy": 0.74, "usd_per_1000": 1.3951, "latency_ms_p50": 22260, "latency_ms_p90": 38174, "invalid_json_rate": 0.1, "by_field": { "supplier_name": 0.86, "supplier_id": 0.89, "vat_id": 0.86, "invoice_number": 0.89, "payment_reference": 0.85, "issue_date": 0.89, "tax_point_date": 0.87, "due_date": 0.89, "net_by_rate": 0.86, "vat_by_rate": 0.9, "total": 0.89, "currency": 0.89, "bank_account": 0.89 }, "errors": { "invalid_json": 130, "missing_field": 14, "hallucinated": 10, "wrong_value": 2, "lost_diacritics": 1 }, "n_cases": 100, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "es", "level": "text", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 5.2156, "latency_ms_p50": 4740, "latency_ms_p90": 6344, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 30, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9974, "doc_accuracy": 0.9667, "usd_per_1000": 0.2333, "latency_ms_p50": 2998, "latency_ms_p90": 4122, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9974, "doc_accuracy": 0.9667, "usd_per_1000": 4.3284, "latency_ms_p50": 4496, "latency_ms_p90": 5754, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9974, "doc_accuracy": 0.9667, "usd_per_1000": 1.1228, "latency_ms_p50": 26587, "latency_ms_p90": 46450, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9904, "doc_accuracy": 0.875, "usd_per_1000": 0.5999, "latency_ms_p50": 7319, "latency_ms_p90": 13076, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.875, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 3 }, "n_cases": 24, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 3.0348, "latency_ms_p50": 1473, "latency_ms_p90": 1671, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.8667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 4 }, "n_cases": 30, "passes_bar": true, "top_error": "missing_field", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9846, "doc_accuracy": 0.8, "usd_per_1000": 0.9366, "latency_ms_p50": 1106, "latency_ms_p90": 1248, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.8, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 6 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9808, "doc_accuracy": 0.75, "usd_per_1000": 0.2358, "latency_ms_p50": 7074, "latency_ms_p90": 9658, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9643, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.7857, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 6, "hallucinated": 1 }, "n_cases": 28, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9615, "doc_accuracy": 0.5417, "usd_per_1000": 0.245, "latency_ms_p50": 1595, "latency_ms_p90": 1822, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.8333, "invoice_number": 1, "payment_reference": 0.9583, "issue_date": 1, "tax_point_date": 0.75, "due_date": 1, "net_by_rate": 0.9583, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 11, "missing_field": 1 }, "n_cases": 24, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9179, "doc_accuracy": 0.7333, "usd_per_1000": 6.8795, "latency_ms_p50": 4852, "latency_ms_p90": 7572, "invalid_json_rate": 0.0667, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 0.9333, "payment_reference": 0.9, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.7667, "vat_by_rate": 0.9333, "total": 0.9333, "currency": 0.9333, "bank_account": 0.9333 }, "errors": { "invalid_json": 26, "missing_field": 5, "hallucinated": 1 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" } ] }, { "task": "invoices", "lang": "es", "level": "clean", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 0.7926, "latency_ms_p50": 4324, "latency_ms_p90": 6150, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8667 }, "errors": { "wrong_value": 4 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 15.4773, "latency_ms_p50": 6621, "latency_ms_p90": 7598, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.8667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 4 }, "n_cases": 30, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9872, "doc_accuracy": 0.8333, "usd_per_1000": 1.0878, "latency_ms_p50": 1833, "latency_ms_p90": 2152, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.8333, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 5 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9829, "doc_accuracy": 0.7778, "usd_per_1000": 0.6743, "latency_ms_p50": 6196, "latency_ms_p90": 14735, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9259, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.8519, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 4, "wrong_value": 1, "lost_diacritics": 1 }, "n_cases": 27, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9692, "doc_accuracy": 0.6667, "usd_per_1000": 5.4235, "latency_ms_p50": 2491, "latency_ms_p90": 4179, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9667, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 0.9333, "due_date": 0.9667, "net_by_rate": 0.8333, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 6, "missing_field": 6 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9402, "doc_accuracy": 0.4074, "usd_per_1000": 0.1877, "latency_ms_p50": 8934, "latency_ms_p90": 11014, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.963, "supplier_id": 0.8519, "vat_id": 0.963, "invoice_number": 1, "payment_reference": 0.963, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.7778, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.7037 }, "errors": { "wrong_value": 15, "missing_field": 6 }, "n_cases": 27, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9333, "doc_accuracy": 0.9333, "usd_per_1000": 8.42, "latency_ms_p50": 6033, "latency_ms_p90": 10371, "invalid_json_rate": 0.0667, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 0.9333, "payment_reference": 0.9333, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.9333, "vat_by_rate": 0.9333, "total": 0.9333, "currency": 0.9333, "bank_account": 0.9333 }, "errors": { "invalid_json": 26 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9179, "doc_accuracy": 0.7333, "usd_per_1000": 15.4133, "latency_ms_p50": 4202, "latency_ms_p90": 6706, "invalid_json_rate": 0.0667, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 0.9333, "payment_reference": 0.9333, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.7333, "vat_by_rate": 0.9333, "total": 0.9333, "currency": 0.9333, "bank_account": 0.9333 }, "errors": { "invalid_json": 26, "missing_field": 6 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.8985, "doc_accuracy": 0.2, "usd_per_1000": 0.5201, "latency_ms_p50": 1958, "latency_ms_p90": 2590, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.44, "invoice_number": 1, "payment_reference": 0.84, "issue_date": 1, "tax_point_date": 0.6, "due_date": 1, "net_by_rate": 0.96, "vat_by_rate": 0.96, "total": 1, "currency": 1, "bank_account": 0.88 }, "errors": { "hallucinated": 28, "wrong_value": 5 }, "n_cases": 25, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8462, "doc_accuracy": 0.5667, "usd_per_1000": 1.4266, "latency_ms_p50": 19984, "latency_ms_p90": 38174, "invalid_json_rate": 0.1, "by_field": { "supplier_name": 0.8333, "supplier_id": 0.8667, "vat_id": 0.8, "invoice_number": 0.8667, "payment_reference": 0.8, "issue_date": 0.8667, "tax_point_date": 0.8, "due_date": 0.8667, "net_by_rate": 0.8, "vat_by_rate": 0.9, "total": 0.8667, "currency": 0.8667, "bank_account": 0.8667 }, "errors": { "invalid_json": 39, "missing_field": 13, "hallucinated": 7, "wrong_value": 1 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "es", "level": "scan", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9872, "doc_accuracy": 0.8333, "usd_per_1000": 0.6486, "latency_ms_p50": 3926, "latency_ms_p90": 5041, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9 }, "errors": { "wrong_value": 4, "hallucinated": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9872, "doc_accuracy": 0.8333, "usd_per_1000": 12.714, "latency_ms_p50": 5833, "latency_ms_p90": 7288, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9667 }, "errors": { "missing_field": 3, "wrong_value": 2 }, "n_cases": 30, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9744, "doc_accuracy": 0.7333, "usd_per_1000": 1.0953, "latency_ms_p50": 1779, "latency_ms_p90": 2426, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9333, "supplier_id": 1, "vat_id": 0.8333, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9667, "tax_point_date": 0.9333, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 6, "wrong_value": 4 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9731, "doc_accuracy": 0.65, "usd_per_1000": 0.635, "latency_ms_p50": 5363, "latency_ms_p90": 15461, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.95, "issue_date": 1, "tax_point_date": 0.95, "due_date": 1, "net_by_rate": 0.9, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.95 }, "errors": { "hallucinated": 2, "missing_field": 2, "wrong_value": 2, "lost_diacritics": 1 }, "n_cases": 20, "passes_bar": true, "top_error": "hallucinated", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9538, "doc_accuracy": 0.5, "usd_per_1000": 5.4533, "latency_ms_p50": 2670, "latency_ms_p90": 5304, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 1, "vat_id": 0.9, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 0.8667, "due_date": 0.9667, "net_by_rate": 0.8, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 10, "missing_field": 7, "wrong_value": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9513, "doc_accuracy": 0.7667, "usd_per_1000": 13.5109, "latency_ms_p50": 3881, "latency_ms_p90": 6961, "invalid_json_rate": 0.0333, "by_field": { "supplier_name": 0.9333, "supplier_id": 0.9667, "vat_id": 0.9667, "invoice_number": 0.9667, "payment_reference": 0.9667, "issue_date": 0.9667, "tax_point_date": 0.9667, "due_date": 0.9667, "net_by_rate": 0.8, "vat_by_rate": 0.9667, "total": 0.9667, "currency": 0.9667, "bank_account": 0.9667 }, "errors": { "invalid_json": 13, "missing_field": 5, "wrong_value": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "invalid_json", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9354, "doc_accuracy": 0.4, "usd_per_1000": 0.1882, "latency_ms_p50": 8870, "latency_ms_p90": 11584, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.92, "supplier_id": 0.92, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.92, "issue_date": 0.96, "tax_point_date": 1, "due_date": 0.96, "net_by_rate": 0.84, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.64 }, "errors": { "wrong_value": 17, "missing_field": 4 }, "n_cases": 25, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9308, "doc_accuracy": 0.9, "usd_per_1000": 10.2156, "latency_ms_p50": 5599, "latency_ms_p90": 13132, "invalid_json_rate": 0.0667, "by_field": { "supplier_name": 0.9, "supplier_id": 0.9333, "vat_id": 0.9333, "invoice_number": 0.9333, "payment_reference": 0.9333, "issue_date": 0.9333, "tax_point_date": 0.9333, "due_date": 0.9333, "net_by_rate": 0.9333, "vat_by_rate": 0.9333, "total": 0.9333, "currency": 0.9333, "bank_account": 0.9333 }, "errors": { "invalid_json": 26, "wrong_value": 1 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.8964, "doc_accuracy": 0.1538, "usd_per_1000": 0.5217, "latency_ms_p50": 2571, "latency_ms_p90": 3117, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9231, "supplier_id": 0.9615, "vat_id": 0.3462, "invoice_number": 1, "payment_reference": 0.8462, "issue_date": 1, "tax_point_date": 0.6923, "due_date": 1, "net_by_rate": 0.9615, "vat_by_rate": 0.9615, "total": 0.9615, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 28, "wrong_value": 6, "swapped_dates": 1 }, "n_cases": 26, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.7949, "doc_accuracy": 0.7333, "usd_per_1000": 1.674, "latency_ms_p50": 20894, "latency_ms_p90": 35161, "invalid_json_rate": 0.2, "by_field": { "supplier_name": 0.7667, "supplier_id": 0.8, "vat_id": 0.8, "invoice_number": 0.8, "payment_reference": 0.8, "issue_date": 0.8, "tax_point_date": 0.8, "due_date": 0.8, "net_by_rate": 0.7667, "vat_by_rate": 0.8, "total": 0.8, "currency": 0.8, "bank_account": 0.8 }, "errors": { "invalid_json": 78, "missing_field": 1, "wrong_value": 1 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "es", "level": "photo", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 11.4632, "latency_ms_p50": 3863, "latency_ms_p90": 4692, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 10, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 7.0838, "latency_ms_p50": 7107, "latency_ms_p90": 12185, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 10, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9923, "doc_accuracy": 0.9, "usd_per_1000": 10.0325, "latency_ms_p50": 6072, "latency_ms_p90": 8558, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 10, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9846, "doc_accuracy": 0.8, "usd_per_1000": 1.1312, "latency_ms_p50": 1836, "latency_ms_p90": 2075, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 2 }, "n_cases": 10, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9846, "doc_accuracy": 0.8, "usd_per_1000": 0.4977, "latency_ms_p50": 4426, "latency_ms_p90": 5774, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8 }, "errors": { "wrong_value": 2 }, "n_cases": 10, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9808, "doc_accuracy": 0.75, "usd_per_1000": 1.3048, "latency_ms_p50": 7650, "latency_ms_p90": 34555, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.875, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.875, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1, "lost_diacritics": 1 }, "n_cases": 8, "passes_bar": true, "top_error": "hallucinated", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.967, "doc_accuracy": 0.7143, "usd_per_1000": 0.1953, "latency_ms_p50": 9250, "latency_ms_p90": 49805, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.8571, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 0.8571, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8571 }, "errors": { "wrong_value": 3 }, "n_cases": 7, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9615, "doc_accuracy": 0.6, "usd_per_1000": 5.7285, "latency_ms_p50": 2920, "latency_ms_p90": 4550, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.8, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 0.9, "due_date": 0.9, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 4, "missing_field": 1 }, "n_cases": 10, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9316, "doc_accuracy": 0.4444, "usd_per_1000": 0.5438, "latency_ms_p50": 2839, "latency_ms_p90": 4022, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.5556, "invoice_number": 1, "payment_reference": 0.8889, "issue_date": 1, "tax_point_date": 0.7778, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8889 }, "errors": { "hallucinated": 5, "swapped_dates": 2, "wrong_value": 1 }, "n_cases": 9, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8769, "doc_accuracy": 0.6, "usd_per_1000": 1.2804, "latency_ms_p50": 17230, "latency_ms_p90": 47315, "invalid_json_rate": 0.1, "by_field": { "supplier_name": 0.8, "supplier_id": 0.9, "vat_id": 0.8, "invoice_number": 0.9, "payment_reference": 0.8, "issue_date": 0.9, "tax_point_date": 0.9, "due_date": 0.9, "net_by_rate": 0.9, "vat_by_rate": 0.9, "total": 0.9, "currency": 0.9, "bank_account": 0.9 }, "errors": { "invalid_json": 13, "hallucinated": 2, "lost_diacritics": 1 }, "n_cases": 10, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "fr", "level": "all", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9992, "doc_accuracy": 0.99, "usd_per_1000": 6.175, "latency_ms_p50": 5157, "latency_ms_p90": 9128, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.99, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 100, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9992, "doc_accuracy": 0.99, "usd_per_1000": 11.0514, "latency_ms_p50": 5874, "latency_ms_p90": 7599, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.99, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 100, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9938, "doc_accuracy": 0.9189, "usd_per_1000": 0.6481, "latency_ms_p50": 5737, "latency_ms_p90": 13604, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9865, "invoice_number": 1, "payment_reference": 0.9865, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9459 }, "errors": { "wrong_value": 5, "hallucinated": 1 }, "n_cases": 74, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.989, "doc_accuracy": 0.8571, "usd_per_1000": 0.5656, "latency_ms_p50": 3817, "latency_ms_p90": 5211, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9898, "supplier_id": 0.9898, "vat_id": 0.9796, "invoice_number": 0.9898, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9082 }, "errors": { "wrong_value": 14 }, "n_cases": 98, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9869, "doc_accuracy": 0.85, "usd_per_1000": 1.0656, "latency_ms_p50": 1514, "latency_ms_p90": 1844, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.98, "supplier_id": 1, "vat_id": 1, "invoice_number": 0.99, "payment_reference": 0.91, "issue_date": 0.98, "tax_point_date": 0.98, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.99 }, "errors": { "hallucinated": 9, "wrong_value": 4, "swapped_dates": 4 }, "n_cases": 100, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9865, "doc_accuracy": 0.9588, "usd_per_1000": 11.0212, "latency_ms_p50": 4366, "latency_ms_p90": 8310, "invalid_json_rate": 0.0103, "by_field": { "supplier_name": 0.9897, "supplier_id": 0.9897, "vat_id": 0.9897, "invoice_number": 0.9897, "payment_reference": 0.9794, "issue_date": 0.9897, "tax_point_date": 0.9897, "due_date": 0.9897, "net_by_rate": 0.9691, "vat_by_rate": 0.9897, "total": 0.9897, "currency": 0.9897, "bank_account": 0.9794 }, "errors": { "invalid_json": 13, "missing_field": 2, "hallucinated": 1, "wrong_value": 1 }, "n_cases": 97, "passes_bar": true, "top_error": "invalid_json", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9815, "doc_accuracy": 0.81, "usd_per_1000": 4.8072, "latency_ms_p50": 2580, "latency_ms_p90": 3864, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.99, "supplier_id": 0.99, "vat_id": 0.98, "invoice_number": 1, "payment_reference": 0.94, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.95, "vat_by_rate": 0.98, "total": 0.99, "currency": 0.96, "bank_account": 0.98 }, "errors": { "wrong_value": 13, "hallucinated": 7, "missing_field": 4 }, "n_cases": 100, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9423, "doc_accuracy": 0.5, "usd_per_1000": 0.4465, "latency_ms_p50": 2103, "latency_ms_p90": 2719, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9605, "supplier_id": 0.9342, "vat_id": 0.9342, "invoice_number": 1, "payment_reference": 0.8553, "issue_date": 1, "tax_point_date": 0.9211, "due_date": 1, "net_by_rate": 0.9474, "vat_by_rate": 1, "total": 1, "currency": 0.9474, "bank_account": 0.75 }, "errors": { "wrong_value": 33, "hallucinated": 23, "swapped_dates": 1 }, "n_cases": 76, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9355, "doc_accuracy": 0.5057, "usd_per_1000": 0.2218, "latency_ms_p50": 7185, "latency_ms_p90": 10891, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9655, "supplier_id": 0.7701, "vat_id": 0.954, "invoice_number": 0.977, "payment_reference": 0.9655, "issue_date": 0.9655, "tax_point_date": 0.9425, "due_date": 0.9655, "net_by_rate": 0.931, "vat_by_rate": 1, "total": 1, "currency": 0.9885, "bank_account": 0.7356 }, "errors": { "wrong_value": 64, "missing_field": 7, "swapped_dates": 2 }, "n_cases": 87, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9062, "doc_accuracy": 0.82, "usd_per_1000": 1.2291, "latency_ms_p50": 19842, "latency_ms_p90": 35110, "invalid_json_rate": 0.05, "by_field": { "supplier_name": 0.91, "supplier_id": 0.91, "vat_id": 0.91, "invoice_number": 0.9, "payment_reference": 0.86, "issue_date": 0.91, "tax_point_date": 0.92, "due_date": 0.89, "net_by_rate": 0.91, "vat_by_rate": 0.94, "total": 0.91, "currency": 0.91, "bank_account": 0.9 }, "errors": { "invalid_json": 65, "missing_field": 49, "hallucinated": 5, "wrong_value": 3 }, "n_cases": 100, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "fr", "level": "text", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 6.227, "latency_ms_p50": 4870, "latency_ms_p90": 7771, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 29, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 5.0002, "latency_ms_p50": 3463, "latency_ms_p90": 6616, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 30, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.3417, "latency_ms_p50": 4249, "latency_ms_p90": 5222, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 30, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9974, "doc_accuracy": 0.9667, "usd_per_1000": 3.1419, "latency_ms_p50": 1516, "latency_ms_p90": 1914, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "missing_field", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9974, "doc_accuracy": 0.9667, "usd_per_1000": 1.0013, "latency_ms_p50": 24352, "latency_ms_p90": 41640, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9973, "doc_accuracy": 0.9643, "usd_per_1000": 0.2335, "latency_ms_p50": 3081, "latency_ms_p90": 4731, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9643 }, "errors": { "wrong_value": 1 }, "n_cases": 28, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9963, "doc_accuracy": 0.9524, "usd_per_1000": 0.7155, "latency_ms_p50": 6122, "latency_ms_p90": 17391, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9524 }, "errors": { "wrong_value": 1 }, "n_cases": 21, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9945, "doc_accuracy": 0.9286, "usd_per_1000": 0.2581, "latency_ms_p50": 6489, "latency_ms_p90": 9401, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9643, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9643 }, "errors": { "wrong_value": 2 }, "n_cases": 28, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9923, "doc_accuracy": 0.9, "usd_per_1000": 0.965, "latency_ms_p50": 1123, "latency_ms_p90": 1268, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 3 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9825, "doc_accuracy": 0.8182, "usd_per_1000": 0.2683, "latency_ms_p50": 1468, "latency_ms_p90": 1776, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9545, "invoice_number": 1, "payment_reference": 0.8636, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9545, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 5 }, "n_cases": 22, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "invoices", "lang": "fr", "level": "clean", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 6.2689, "latency_ms_p50": 5347, "latency_ms_p90": 9205, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 30, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 15.4335, "latency_ms_p50": 6112, "latency_ms_p90": 7684, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 30, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9973, "doc_accuracy": 0.9655, "usd_per_1000": 13.2247, "latency_ms_p50": 4236, "latency_ms_p90": 8310, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9655, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "missing_field": 1 }, "n_cases": 29, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.993, "doc_accuracy": 0.9091, "usd_per_1000": 0.5977, "latency_ms_p50": 4972, "latency_ms_p90": 9320, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9091 }, "errors": { "wrong_value": 2 }, "n_cases": 22, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9846, "doc_accuracy": 0.8, "usd_per_1000": 1.0959, "latency_ms_p50": 1642, "latency_ms_p90": 2004, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 1, "vat_id": 1, "invoice_number": 0.9667, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9667 }, "errors": { "wrong_value": 3, "hallucinated": 3 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9795, "doc_accuracy": 0.7333, "usd_per_1000": 0.7835, "latency_ms_p50": 4250, "latency_ms_p90": 5459, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.7667 }, "errors": { "wrong_value": 8 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9769, "doc_accuracy": 0.7333, "usd_per_1000": 5.4665, "latency_ms_p50": 2604, "latency_ms_p90": 5998, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 0.9667, "vat_id": 0.9333, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 0.9667, "bank_account": 0.9667 }, "errors": { "wrong_value": 5, "hallucinated": 4 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9138, "doc_accuracy": 0.32, "usd_per_1000": 0.5145, "latency_ms_p50": 1999, "latency_ms_p90": 2562, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.96, "supplier_id": 0.88, "vat_id": 0.92, "invoice_number": 1, "payment_reference": 0.88, "issue_date": 1, "tax_point_date": 0.88, "due_date": 1, "net_by_rate": 0.88, "vat_by_rate": 1, "total": 1, "currency": 0.92, "bank_account": 0.56 }, "errors": { "wrong_value": 17, "hallucinated": 11 }, "n_cases": 25, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9135, "doc_accuracy": 0.2917, "usd_per_1000": 0.2015, "latency_ms_p50": 7893, "latency_ms_p90": 10655, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9167, "supplier_id": 0.5833, "vat_id": 0.9583, "invoice_number": 0.9583, "payment_reference": 1, "issue_date": 1, "tax_point_date": 0.9167, "due_date": 1, "net_by_rate": 0.875, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.6667 }, "errors": { "wrong_value": 25, "missing_field": 2 }, "n_cases": 24, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8359, "doc_accuracy": 0.7667, "usd_per_1000": 1.2627, "latency_ms_p50": 17811, "latency_ms_p90": 31208, "invalid_json_rate": 0.1, "by_field": { "supplier_name": 0.8333, "supplier_id": 0.8333, "vat_id": 0.8333, "invoice_number": 0.8, "payment_reference": 0.8, "issue_date": 0.8333, "tax_point_date": 0.8667, "due_date": 0.8333, "net_by_rate": 0.8333, "vat_by_rate": 0.9, "total": 0.8333, "currency": 0.8333, "bank_account": 0.8333 }, "errors": { "invalid_json": 39, "missing_field": 22, "wrong_value": 2, "hallucinated": 1 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "fr", "level": "scan", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 13.6762, "latency_ms_p50": 6192, "latency_ms_p90": 7906, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 30, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9974, "doc_accuracy": 0.9667, "usd_per_1000": 6.8685, "latency_ms_p50": 5924, "latency_ms_p90": 9433, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9936, "doc_accuracy": 0.9167, "usd_per_1000": 0.6538, "latency_ms_p50": 5804, "latency_ms_p90": 11754, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9583, "invoice_number": 1, "payment_reference": 0.9583, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 1, "wrong_value": 1 }, "n_cases": 24, "passes_bar": true, "top_error": "hallucinated", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9923, "doc_accuracy": 0.9333, "usd_per_1000": 12.9166, "latency_ms_p50": 4476, "latency_ms_p90": 11382, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9667, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9667, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9667 }, "errors": { "missing_field": 1, "hallucinated": 1, "wrong_value": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9897, "doc_accuracy": 0.8667, "usd_per_1000": 0.6806, "latency_ms_p50": 4021, "latency_ms_p90": 5274, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.9333, "invoice_number": 0.9667, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9667 }, "errors": { "wrong_value": 4 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9846, "doc_accuracy": 0.8333, "usd_per_1000": 1.1119, "latency_ms_p50": 1604, "latency_ms_p90": 1933, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9667, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 0.9667, "tax_point_date": 0.9667, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "hallucinated": 3, "swapped_dates": 2, "wrong_value": 1 }, "n_cases": 30, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9744, "doc_accuracy": 0.7333, "usd_per_1000": 5.4967, "latency_ms_p50": 2960, "latency_ms_p90": 3864, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9, "vat_by_rate": 1, "total": 1, "currency": 0.9, "bank_account": 0.9667 }, "errors": { "wrong_value": 4, "missing_field": 3, "hallucinated": 3 }, "n_cases": 30, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9336, "doc_accuracy": 0.4091, "usd_per_1000": 0.5178, "latency_ms_p50": 2526, "latency_ms_p90": 3038, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.9091, "supplier_id": 0.9091, "vat_id": 0.9545, "invoice_number": 1, "payment_reference": 0.8182, "issue_date": 1, "tax_point_date": 0.9091, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 0.9091, "bank_account": 0.7273 }, "errors": { "wrong_value": 13, "hallucinated": 6 }, "n_cases": 22, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9262, "doc_accuracy": 0.36, "usd_per_1000": 0.2052, "latency_ms_p50": 7696, "latency_ms_p90": 11658, "invalid_json_rate": 0, "by_field": { "supplier_name": 0.96, "supplier_id": 0.72, "vat_id": 0.96, "invoice_number": 1, "payment_reference": 0.96, "issue_date": 0.96, "tax_point_date": 0.96, "due_date": 1, "net_by_rate": 0.92, "vat_by_rate": 1, "total": 1, "currency": 0.96, "bank_account": 0.64 }, "errors": { "wrong_value": 19, "missing_field": 3, "swapped_dates": 2 }, "n_cases": 25, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.8615, "doc_accuracy": 0.7667, "usd_per_1000": 1.3844, "latency_ms_p50": 18975, "latency_ms_p90": 34797, "invalid_json_rate": 0.0667, "by_field": { "supplier_name": 0.8667, "supplier_id": 0.8667, "vat_id": 0.8667, "invoice_number": 0.8667, "payment_reference": 0.8, "issue_date": 0.8667, "tax_point_date": 0.8667, "due_date": 0.8333, "net_by_rate": 0.8667, "vat_by_rate": 0.9, "total": 0.8667, "currency": 0.8667, "bank_account": 0.8667 }, "errors": { "invalid_json": 26, "missing_field": 26, "hallucinated": 2 }, "n_cases": 30, "passes_bar": false, "top_error": "invalid_json", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "invoices", "lang": "fr", "level": "photo", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 7.3376, "latency_ms_p50": 7206, "latency_ms_p90": 9278, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": {}, "n_cases": 10, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9923, "doc_accuracy": 0.9, "usd_per_1000": 0.4973, "latency_ms_p50": 3715, "latency_ms_p90": 4945, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.9, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 10, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9923, "doc_accuracy": 0.9, "usd_per_1000": 10.16, "latency_ms_p50": 6752, "latency_ms_p90": 9928, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 10, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.989, "doc_accuracy": 0.8571, "usd_per_1000": 0.5851, "latency_ms_p50": 5820, "latency_ms_p90": 9139, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.8571 }, "errors": { "wrong_value": 1 }, "n_cases": 7, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9846, "doc_accuracy": 0.9, "usd_per_1000": 1.138, "latency_ms_p50": 1514, "latency_ms_p90": 1767, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 0.9, "tax_point_date": 0.9, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 1 }, "errors": { "swapped_dates": 2 }, "n_cases": 10, "passes_bar": true, "top_error": "swapped_dates", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9769, "doc_accuracy": 0.7, "usd_per_1000": 1.3457, "latency_ms_p50": 20764, "latency_ms_p90": 41787, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 0.9, "issue_date": 1, "tax_point_date": 1, "due_date": 0.9, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.9 }, "errors": { "wrong_value": 1, "hallucinated": 1, "missing_field": 1 }, "n_cases": 10, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9692, "doc_accuracy": 0.8, "usd_per_1000": 5.757, "latency_ms_p50": 2932, "latency_ms_p90": 10840, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 1, "invoice_number": 1, "payment_reference": 1, "issue_date": 1, "tax_point_date": 1, "due_date": 1, "net_by_rate": 0.9, "vat_by_rate": 0.8, "total": 0.9, "currency": 1, "bank_account": 1 }, "errors": { "wrong_value": 4 }, "n_cases": 10, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9451, "doc_accuracy": 0.4286, "usd_per_1000": 0.54, "latency_ms_p50": 2653, "latency_ms_p90": 2957, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 1, "vat_id": 0.8571, "invoice_number": 1, "payment_reference": 0.8571, "issue_date": 1, "tax_point_date": 0.8571, "due_date": 1, "net_by_rate": 1, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.7143 }, "errors": { "wrong_value": 3, "hallucinated": 1, "swapped_dates": 1 }, "n_cases": 7, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.8889, "doc_accuracy": 0.8889, "usd_per_1000": 13.0511, "latency_ms_p50": 5444, "latency_ms_p90": 13861, "invalid_json_rate": 0.1111, "by_field": { "supplier_name": 0.8889, "supplier_id": 0.8889, "vat_id": 0.8889, "invoice_number": 0.8889, "payment_reference": 0.8889, "issue_date": 0.8889, "tax_point_date": 0.8889, "due_date": 0.8889, "net_by_rate": 0.8889, "vat_by_rate": 0.8889, "total": 0.8889, "currency": 0.8889, "bank_account": 0.8889 }, "errors": { "invalid_json": 13 }, "n_cases": 9, "passes_bar": false, "top_error": "invalid_json", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.8462, "doc_accuracy": 0.2, "usd_per_1000": 0.2105, "latency_ms_p50": 8460, "latency_ms_p90": 11644, "invalid_json_rate": 0, "by_field": { "supplier_name": 1, "supplier_id": 0.8, "vat_id": 0.8, "invoice_number": 0.9, "payment_reference": 0.8, "issue_date": 0.8, "tax_point_date": 0.8, "due_date": 0.7, "net_by_rate": 0.9, "vat_by_rate": 1, "total": 1, "currency": 1, "bank_account": 0.5 }, "errors": { "wrong_value": 18, "missing_field": 2 }, "n_cases": 10, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" } ] }, { "task": "emails", "lang": "cs", "level": "all", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9919, "doc_accuracy": 0.9267, "usd_per_1000": 3.0603, "latency_ms_p50": 2583, "latency_ms_p90": 3789, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9933, "sentiment": 0.9333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 11 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9896, "doc_accuracy": 0.9067, "usd_per_1000": 3.1916, "latency_ms_p50": 2819, "latency_ms_p90": 4298, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.98, "sentiment": 0.9267, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 14 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9852, "doc_accuracy": 0.8667, "usd_per_1000": 0.4453, "latency_ms_p50": 12968, "latency_ms_p90": 22193, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9933, "sentiment": 0.8867, "customer_name": 0.9933, "order_number": 0.9933, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 20 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9815, "doc_accuracy": 0.8333, "usd_per_1000": 4.9087, "latency_ms_p50": 4678, "latency_ms_p90": 7113, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.96, "sentiment": 0.8733, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 25 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9793, "doc_accuracy": 0.8133, "usd_per_1000": 0.1781, "latency_ms_p50": 2554, "latency_ms_p90": 3123, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9867, "sentiment": 0.8267, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 28 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9741, "doc_accuracy": 0.7733, "usd_per_1000": 0.2957, "latency_ms_p50": 4152, "latency_ms_p90": 7361, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9867, "sentiment": 0.78, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 35 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9711, "doc_accuracy": 0.7467, "usd_per_1000": 0.1589, "latency_ms_p50": 2514, "latency_ms_p90": 5260, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.8533, "sentiment": 0.8867, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 39 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9578, "doc_accuracy": 0.64, "usd_per_1000": 0.5091, "latency_ms_p50": 731, "latency_ms_p90": 823, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.8, "sentiment": 0.84, "customer_name": 0.98, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 57 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9452, "doc_accuracy": 0.5533, "usd_per_1000": 1.9474, "latency_ms_p50": 822, "latency_ms_p90": 1112, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.6333, "sentiment": 0.8733, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 74 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.877, "doc_accuracy": 0.3154, "usd_per_1000": 0.1177, "latency_ms_p50": 811, "latency_ms_p90": 1189, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.5235, "sentiment": 0.7248, "customer_name": 0.9866, "order_number": 1, "invoice_number": 1, "amount": 0.9799, "currency": 0.9799, "deadline": 0.698 }, "errors": { "wrong_value": 115, "hallucinated": 44, "missing_field": 6 }, "n_cases": 149, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "emails", "lang": "cs", "level": "text", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9919, "doc_accuracy": 0.9267, "usd_per_1000": 3.0603, "latency_ms_p50": 2583, "latency_ms_p90": 3789, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9933, "sentiment": 0.9333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 11 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9896, "doc_accuracy": 0.9067, "usd_per_1000": 3.1916, "latency_ms_p50": 2819, "latency_ms_p90": 4298, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.98, "sentiment": 0.9267, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 14 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9852, "doc_accuracy": 0.8667, "usd_per_1000": 0.4453, "latency_ms_p50": 12968, "latency_ms_p90": 22193, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9933, "sentiment": 0.8867, "customer_name": 0.9933, "order_number": 0.9933, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 20 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9815, "doc_accuracy": 0.8333, "usd_per_1000": 4.9087, "latency_ms_p50": 4678, "latency_ms_p90": 7113, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.96, "sentiment": 0.8733, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 25 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9793, "doc_accuracy": 0.8133, "usd_per_1000": 0.1781, "latency_ms_p50": 2554, "latency_ms_p90": 3123, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9867, "sentiment": 0.8267, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 28 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9741, "doc_accuracy": 0.7733, "usd_per_1000": 0.2957, "latency_ms_p50": 4152, "latency_ms_p90": 7361, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9867, "sentiment": 0.78, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 35 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9711, "doc_accuracy": 0.7467, "usd_per_1000": 0.1589, "latency_ms_p50": 2514, "latency_ms_p90": 5260, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.8533, "sentiment": 0.8867, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 39 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9578, "doc_accuracy": 0.64, "usd_per_1000": 0.5091, "latency_ms_p50": 731, "latency_ms_p90": 823, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.8, "sentiment": 0.84, "customer_name": 0.98, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 57 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9452, "doc_accuracy": 0.5533, "usd_per_1000": 1.9474, "latency_ms_p50": 822, "latency_ms_p90": 1112, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.6333, "sentiment": 0.8733, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 74 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.877, "doc_accuracy": 0.3154, "usd_per_1000": 0.1177, "latency_ms_p50": 811, "latency_ms_p90": 1189, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.5235, "sentiment": 0.7248, "customer_name": 0.9866, "order_number": 1, "invoice_number": 1, "amount": 0.9799, "currency": 0.9799, "deadline": 0.698 }, "errors": { "wrong_value": 115, "hallucinated": 44, "missing_field": 6 }, "n_cases": 149, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "emails", "lang": "de", "level": "all", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9837, "doc_accuracy": 0.8533, "usd_per_1000": 2.743, "latency_ms_p50": 2862, "latency_ms_p90": 4287, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 1, "sentiment": 0.8533, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 22 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9822, "doc_accuracy": 0.84, "usd_per_1000": 2.729, "latency_ms_p50": 2939, "latency_ms_p90": 4115, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9733, "sentiment": 0.8667, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 24 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9785, "doc_accuracy": 0.8133, "usd_per_1000": 5.1127, "latency_ms_p50": 4312, "latency_ms_p90": 10821, "invalid_json_rate": 0, "by_field": { "currency": 1, "deadline": 1, "category": 1, "priority": 0.96, "sentiment": 0.8467, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1 }, "errors": { "wrong_value": 29 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9785, "doc_accuracy": 0.8133, "usd_per_1000": 0.3915, "latency_ms_p50": 12071, "latency_ms_p90": 18672, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.96, "sentiment": 0.8467, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 29 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9696, "doc_accuracy": 0.7267, "usd_per_1000": 0.1524, "latency_ms_p50": 2776, "latency_ms_p90": 3503, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9933, "sentiment": 0.7333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 41 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9632, "doc_accuracy": 0.6759, "usd_per_1000": 0.3032, "latency_ms_p50": 4157, "latency_ms_p90": 7119, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9724, "sentiment": 0.6966, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 48 }, "n_cases": 145, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9511, "doc_accuracy": 0.62, "usd_per_1000": 0.1483, "latency_ms_p50": 3591, "latency_ms_p90": 5135, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.78, "sentiment": 0.78, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 66 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9467, "doc_accuracy": 0.5667, "usd_per_1000": 0.4767, "latency_ms_p50": 766, "latency_ms_p90": 854, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.76, "sentiment": 0.7667, "customer_name": 0.9933, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 72 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9274, "doc_accuracy": 0.48, "usd_per_1000": 1.7705, "latency_ms_p50": 843, "latency_ms_p90": 1126, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.5667, "sentiment": 0.78, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 98 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.871, "doc_accuracy": 0.2937, "usd_per_1000": 0.1068, "latency_ms_p50": 809, "latency_ms_p90": 1105, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.5524, "sentiment": 0.6503, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 0.965, "currency": 0.965, "deadline": 0.7063 }, "errors": { "wrong_value": 114, "hallucinated": 46, "missing_field": 6 }, "n_cases": 143, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "emails", "lang": "de", "level": "text", "bar": 0.95, "rows": [ { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9837, "doc_accuracy": 0.8533, "usd_per_1000": 2.743, "latency_ms_p50": 2862, "latency_ms_p90": 4287, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 1, "sentiment": 0.8533, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 22 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9822, "doc_accuracy": 0.84, "usd_per_1000": 2.729, "latency_ms_p50": 2939, "latency_ms_p90": 4115, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9733, "sentiment": 0.8667, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 24 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9785, "doc_accuracy": 0.8133, "usd_per_1000": 5.1127, "latency_ms_p50": 4312, "latency_ms_p90": 10821, "invalid_json_rate": 0, "by_field": { "currency": 1, "deadline": 1, "category": 1, "priority": 0.96, "sentiment": 0.8467, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1 }, "errors": { "wrong_value": 29 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9785, "doc_accuracy": 0.8133, "usd_per_1000": 0.3915, "latency_ms_p50": 12071, "latency_ms_p90": 18672, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.96, "sentiment": 0.8467, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 29 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9696, "doc_accuracy": 0.7267, "usd_per_1000": 0.1524, "latency_ms_p50": 2776, "latency_ms_p90": 3503, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9933, "sentiment": 0.7333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 41 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9632, "doc_accuracy": 0.6759, "usd_per_1000": 0.3032, "latency_ms_p50": 4157, "latency_ms_p90": 7119, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9724, "sentiment": 0.6966, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 48 }, "n_cases": 145, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9511, "doc_accuracy": 0.62, "usd_per_1000": 0.1483, "latency_ms_p50": 3591, "latency_ms_p90": 5135, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.78, "sentiment": 0.78, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 66 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9467, "doc_accuracy": 0.5667, "usd_per_1000": 0.4767, "latency_ms_p50": 766, "latency_ms_p90": 854, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.76, "sentiment": 0.7667, "customer_name": 0.9933, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 72 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9274, "doc_accuracy": 0.48, "usd_per_1000": 1.7705, "latency_ms_p50": 843, "latency_ms_p90": 1126, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.5667, "sentiment": 0.78, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 98 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.871, "doc_accuracy": 0.2937, "usd_per_1000": 0.1068, "latency_ms_p50": 809, "latency_ms_p90": 1105, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.5524, "sentiment": 0.6503, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 0.965, "currency": 0.965, "deadline": 0.7063 }, "errors": { "wrong_value": 114, "hallucinated": 46, "missing_field": 6 }, "n_cases": 143, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "emails", "lang": "en", "level": "all", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9904, "doc_accuracy": 0.92, "usd_per_1000": 2.4361, "latency_ms_p50": 2793, "latency_ms_p90": 4032, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.98, "sentiment": 0.9333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 13 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9874, "doc_accuracy": 0.8867, "usd_per_1000": 2.3857, "latency_ms_p50": 2161, "latency_ms_p90": 3312, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 1, "sentiment": 0.8867, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 17 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9822, "doc_accuracy": 0.84, "usd_per_1000": 4.0088, "latency_ms_p50": 3781, "latency_ms_p90": 5478, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9733, "sentiment": 0.8667, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 24 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9763, "doc_accuracy": 0.7867, "usd_per_1000": 0.2328, "latency_ms_p50": 3832, "latency_ms_p90": 6583, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9667, "sentiment": 0.82, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 32 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9748, "doc_accuracy": 0.7733, "usd_per_1000": 0.1347, "latency_ms_p50": 2529, "latency_ms_p90": 3081, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.98, "sentiment": 0.7933, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 34 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9748, "doc_accuracy": 0.7867, "usd_per_1000": 0.3042, "latency_ms_p50": 8894, "latency_ms_p90": 15462, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9733, "sentiment": 0.8067, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 0.9933 }, "errors": { "wrong_value": 33, "hallucinated": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9652, "doc_accuracy": 0.72, "usd_per_1000": 0.1207, "latency_ms_p50": 2210, "latency_ms_p90": 3749, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.8533, "sentiment": 0.8333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 47 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.963, "doc_accuracy": 0.6867, "usd_per_1000": 0.4187, "latency_ms_p50": 715, "latency_ms_p90": 852, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.84, "sentiment": 0.8267, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 50 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9489, "doc_accuracy": 0.5733, "usd_per_1000": 1.4742, "latency_ms_p50": 760, "latency_ms_p90": 1033, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.6333, "sentiment": 0.9067, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 69 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.897, "doc_accuracy": 0.3733, "usd_per_1000": 0.0949, "latency_ms_p50": 807, "latency_ms_p90": 1020, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.6133, "sentiment": 0.7133, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 0.9933, "currency": 0.9933, "deadline": 0.76 }, "errors": { "wrong_value": 103, "hallucinated": 34, "missing_field": 2 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "emails", "lang": "en", "level": "text", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9904, "doc_accuracy": 0.92, "usd_per_1000": 2.4361, "latency_ms_p50": 2793, "latency_ms_p90": 4032, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.98, "sentiment": 0.9333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 13 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9874, "doc_accuracy": 0.8867, "usd_per_1000": 2.3857, "latency_ms_p50": 2161, "latency_ms_p90": 3312, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 1, "sentiment": 0.8867, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 17 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9822, "doc_accuracy": 0.84, "usd_per_1000": 4.0088, "latency_ms_p50": 3781, "latency_ms_p90": 5478, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9733, "sentiment": 0.8667, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 24 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9763, "doc_accuracy": 0.7867, "usd_per_1000": 0.2328, "latency_ms_p50": 3832, "latency_ms_p90": 6583, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9667, "sentiment": 0.82, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 32 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9748, "doc_accuracy": 0.7733, "usd_per_1000": 0.1347, "latency_ms_p50": 2529, "latency_ms_p90": 3081, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.98, "sentiment": 0.7933, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 34 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9748, "doc_accuracy": 0.7867, "usd_per_1000": 0.3042, "latency_ms_p50": 8894, "latency_ms_p90": 15462, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.9733, "sentiment": 0.8067, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 0.9933 }, "errors": { "wrong_value": 33, "hallucinated": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9652, "doc_accuracy": 0.72, "usd_per_1000": 0.1207, "latency_ms_p50": 2210, "latency_ms_p90": 3749, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.8533, "sentiment": 0.8333, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 47 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.963, "doc_accuracy": 0.6867, "usd_per_1000": 0.4187, "latency_ms_p50": 715, "latency_ms_p90": 852, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.84, "sentiment": 0.8267, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 50 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9489, "doc_accuracy": 0.5733, "usd_per_1000": 1.4742, "latency_ms_p50": 760, "latency_ms_p90": 1033, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.6333, "sentiment": 0.9067, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 1, "currency": 1, "deadline": 1 }, "errors": { "wrong_value": 69 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.897, "doc_accuracy": 0.3733, "usd_per_1000": 0.0949, "latency_ms_p50": 807, "latency_ms_p90": 1020, "invalid_json_rate": 0, "by_field": { "category": 1, "priority": 0.6133, "sentiment": 0.7133, "customer_name": 1, "order_number": 1, "invoice_number": 1, "amount": 0.9933, "currency": 0.9933, "deadline": 0.76 }, "errors": { "wrong_value": 103, "hallucinated": 34, "missing_field": 2 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "contracts", "lang": "cs", "level": "all", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 6.0522, "latency_ms_p50": 3577, "latency_ms_p90": 4968, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.4195, "latency_ms_p50": 4273, "latency_ms_p90": 7378, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.0902, "latency_ms_p50": 2237, "latency_ms_p90": 3206, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.0563, "latency_ms_p50": 1183, "latency_ms_p90": 1537, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.27, "latency_ms_p50": 2557, "latency_ms_p90": 3083, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.6242, "latency_ms_p50": 2044, "latency_ms_p90": 2698, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9995, "doc_accuracy": 0.9933, "usd_per_1000": 0.2541, "latency_ms_p50": 6019, "latency_ms_p90": 10114, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9933, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9867, "usd_per_1000": 0.6301, "latency_ms_p50": 13621, "latency_ms_p90": 20470, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 0.9867, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9985, "doc_accuracy": 0.98, "usd_per_1000": 0.8518, "latency_ms_p50": 929, "latency_ms_p90": 1073, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9933, "provider_id": 1, "client_name": 0.9933, "client_id": 1, "contract_type": 0.9933, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 2, "lost_diacritics": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9903, "doc_accuracy": 0.92, "usd_per_1000": 0.2268, "latency_ms_p50": 1154, "latency_ms_p90": 1460, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9533, "provider_id": 1, "client_name": 0.9533, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9667, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 16, "swapped_dates": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "contracts", "lang": "cs", "level": "text", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 6.0522, "latency_ms_p50": 3577, "latency_ms_p90": 4968, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.4195, "latency_ms_p50": 4273, "latency_ms_p90": 7378, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.0902, "latency_ms_p50": 2237, "latency_ms_p90": 3206, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.0563, "latency_ms_p50": 1183, "latency_ms_p90": 1537, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.27, "latency_ms_p50": 2557, "latency_ms_p90": 3083, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.6242, "latency_ms_p50": 2044, "latency_ms_p90": 2698, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9995, "doc_accuracy": 0.9933, "usd_per_1000": 0.2541, "latency_ms_p50": 6019, "latency_ms_p90": 10114, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9933, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9867, "usd_per_1000": 0.6301, "latency_ms_p50": 13621, "latency_ms_p90": 20470, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 0.9867, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9985, "doc_accuracy": 0.98, "usd_per_1000": 0.8518, "latency_ms_p50": 929, "latency_ms_p90": 1073, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9933, "provider_id": 1, "client_name": 0.9933, "client_id": 1, "contract_type": 0.9933, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 2, "lost_diacritics": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9903, "doc_accuracy": 0.92, "usd_per_1000": 0.2268, "latency_ms_p50": 1154, "latency_ms_p90": 1460, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9533, "provider_id": 1, "client_name": 0.9533, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9667, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 16, "swapped_dates": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "contracts", "lang": "de", "level": "all", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.2663, "latency_ms_p50": 3021, "latency_ms_p90": 3846, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.3558, "latency_ms_p50": 2933, "latency_ms_p90": 4171, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 3.4159, "latency_ms_p50": 2618, "latency_ms_p90": 3849, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.9933, "client_name": 1, "client_id": 0.9933, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9821, "doc_accuracy": 0.8667, "usd_per_1000": 7.1056, "latency_ms_p50": 4322, "latency_ms_p90": 9847, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.8733, "client_name": 1, "client_id": 0.8933, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 35 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.98, "doc_accuracy": 0.84, "usd_per_1000": 0.7911, "latency_ms_p50": 974, "latency_ms_p90": 1057, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.86, "client_name": 1, "client_id": 0.8867, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 0.9933, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 38, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.98, "doc_accuracy": 0.8733, "usd_per_1000": 0.7798, "latency_ms_p50": 19706, "latency_ms_p90": 38812, "invalid_json_rate": 0.0067, "by_field": { "provider_name": 0.9933, "provider_id": 0.9467, "client_name": 0.9933, "client_id": 0.9467, "contract_type": 0.9933, "price": 0.9933, "currency": 0.9933, "price_period": 0.9933, "effective_date": 0.9933, "end_date": 0.9933, "notice_period_months": 0.9933, "penalty_pct_per_day": 0.9933, "governing_law": 0.9133 }, "errors": { "hallucinated": 14, "invalid_json": 13, "missing_field": 12 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9724, "doc_accuracy": 0.7724, "usd_per_1000": 0.5113, "latency_ms_p50": 4648, "latency_ms_p90": 9792, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.8138, "client_name": 1, "client_id": 0.8276, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 52 }, "n_cases": 145, "passes_bar": true, "top_error": "hallucinated", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9713, "doc_accuracy": 0.7533, "usd_per_1000": 2.7624, "latency_ms_p50": 1249, "latency_ms_p90": 1606, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.82, "client_name": 1, "client_id": 0.8067, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 43, "missing_field": 9, "wrong_value": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9441, "doc_accuracy": 0.2733, "usd_per_1000": 0.2203, "latency_ms_p50": 2603, "latency_ms_p90": 6307, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 0.2733, "governing_law": 1 }, "errors": { "number_format": 109 }, "n_cases": 150, "passes_bar": false, "top_error": "number_format", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.94, "doc_accuracy": 0.4468, "usd_per_1000": 0.2057, "latency_ms_p50": 1226, "latency_ms_p90": 1614, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9858, "provider_id": 0.6241, "client_name": 0.9858, "client_id": 0.6312, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9929, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 99, "wrong_value": 11 }, "n_cases": 141, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "contracts", "lang": "de", "level": "text", "bar": 0.95, "rows": [ { "model_id": "openai/gpt-6-luna", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.2663, "latency_ms_p50": 3021, "latency_ms_p90": 3846, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 4.3558, "latency_ms_p50": 2933, "latency_ms_p90": 4171, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 3.4159, "latency_ms_p50": 2618, "latency_ms_p90": 3849, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.9933, "client_name": 1, "client_id": 0.9933, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9821, "doc_accuracy": 0.8667, "usd_per_1000": 7.1056, "latency_ms_p50": 4322, "latency_ms_p90": 9847, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.8733, "client_name": 1, "client_id": 0.8933, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 35 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.98, "doc_accuracy": 0.84, "usd_per_1000": 0.7911, "latency_ms_p50": 974, "latency_ms_p90": 1057, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.86, "client_name": 1, "client_id": 0.8867, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 0.9933, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 38, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.98, "doc_accuracy": 0.8733, "usd_per_1000": 0.7798, "latency_ms_p50": 19706, "latency_ms_p90": 38812, "invalid_json_rate": 0.0067, "by_field": { "provider_name": 0.9933, "provider_id": 0.9467, "client_name": 0.9933, "client_id": 0.9467, "contract_type": 0.9933, "price": 0.9933, "currency": 0.9933, "price_period": 0.9933, "effective_date": 0.9933, "end_date": 0.9933, "notice_period_months": 0.9933, "penalty_pct_per_day": 0.9933, "governing_law": 0.9133 }, "errors": { "hallucinated": 14, "invalid_json": 13, "missing_field": 12 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9724, "doc_accuracy": 0.7724, "usd_per_1000": 0.5113, "latency_ms_p50": 4648, "latency_ms_p90": 9792, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.8138, "client_name": 1, "client_id": 0.8276, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 52 }, "n_cases": 145, "passes_bar": true, "top_error": "hallucinated", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9713, "doc_accuracy": 0.7533, "usd_per_1000": 2.7624, "latency_ms_p50": 1249, "latency_ms_p90": 1606, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 0.82, "client_name": 1, "client_id": 0.8067, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 43, "missing_field": 9, "wrong_value": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9441, "doc_accuracy": 0.2733, "usd_per_1000": 0.2203, "latency_ms_p50": 2603, "latency_ms_p90": 6307, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 0.2733, "governing_law": 1 }, "errors": { "number_format": 109 }, "n_cases": 150, "passes_bar": false, "top_error": "number_format", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.94, "doc_accuracy": 0.4468, "usd_per_1000": 0.2057, "latency_ms_p50": 1226, "latency_ms_p90": 1614, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9858, "provider_id": 0.6241, "client_name": 0.9858, "client_id": 0.6312, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9929, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "hallucinated": 99, "wrong_value": 11 }, "n_cases": 141, "passes_bar": false, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "contracts", "lang": "en", "level": "all", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 5.039, "latency_ms_p50": 3395, "latency_ms_p90": 5890, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.3201, "latency_ms_p50": 3879, "latency_ms_p90": 6090, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.3295, "latency_ms_p50": 2483, "latency_ms_p90": 3443, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9995, "doc_accuracy": 0.9933, "usd_per_1000": 0.692, "latency_ms_p50": 951, "latency_ms_p90": 1119, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 0.9933, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 2.4657, "latency_ms_p50": 2031, "latency_ms_p90": 3157, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9933, "provider_id": 1, "client_name": 0.9933, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9979, "doc_accuracy": 0.9867, "usd_per_1000": 0.201, "latency_ms_p50": 2715, "latency_ms_p90": 3285, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9867, "provider_id": 1, "client_name": 0.9867, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9964, "doc_accuracy": 0.9733, "usd_per_1000": 0.18, "latency_ms_p50": 2507, "latency_ms_p90": 6551, "invalid_json_rate": 0, "by_field": { "provider_name": 0.98, "provider_id": 1, "client_name": 0.98, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9933, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 7 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.981, "doc_accuracy": 0.8733, "usd_per_1000": 2.3489, "latency_ms_p50": 1123, "latency_ms_p90": 1425, "invalid_json_rate": 0, "by_field": { "provider_name": 0.8733, "provider_id": 1, "client_name": 0.8867, "client_id": 0.9933, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 36, "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9513, "doc_accuracy": 0.6933, "usd_per_1000": 0.6184, "latency_ms_p50": 14064, "latency_ms_p90": 23066, "invalid_json_rate": 0, "by_field": { "provider_name": 0.6933, "provider_id": 1, "client_name": 0.6933, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9933, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 0.9933, "governing_law": 0.9933 }, "errors": { "wrong_value": 92, "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9154, "doc_accuracy": 0.4333, "usd_per_1000": 0.17, "latency_ms_p50": 1127, "latency_ms_p90": 1407, "invalid_json_rate": 0, "by_field": { "provider_name": 0.4333, "provider_id": 1, "client_name": 0.4667, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 165 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "contracts", "lang": "en", "level": "text", "bar": 0.95, "rows": [ { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 5.039, "latency_ms_p50": 3395, "latency_ms_p90": 5890, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 0.3201, "latency_ms_p50": 3879, "latency_ms_p90": 6090, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 3.3295, "latency_ms_p50": 2483, "latency_ms_p90": 3443, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9995, "doc_accuracy": 0.9933, "usd_per_1000": 0.692, "latency_ms_p50": 951, "latency_ms_p90": 1119, "invalid_json_rate": 0, "by_field": { "provider_name": 1, "provider_id": 1, "client_name": 1, "client_id": 1, "contract_type": 0.9933, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 2.4657, "latency_ms_p50": 2031, "latency_ms_p90": 3157, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9933, "provider_id": 1, "client_name": 0.9933, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9979, "doc_accuracy": 0.9867, "usd_per_1000": 0.201, "latency_ms_p50": 2715, "latency_ms_p90": 3285, "invalid_json_rate": 0, "by_field": { "provider_name": 0.9867, "provider_id": 1, "client_name": 0.9867, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9964, "doc_accuracy": 0.9733, "usd_per_1000": 0.18, "latency_ms_p50": 2507, "latency_ms_p90": 6551, "invalid_json_rate": 0, "by_field": { "provider_name": 0.98, "provider_id": 1, "client_name": 0.98, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9933, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 7 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.981, "doc_accuracy": 0.8733, "usd_per_1000": 2.3489, "latency_ms_p50": 1123, "latency_ms_p90": 1425, "invalid_json_rate": 0, "by_field": { "provider_name": 0.8733, "provider_id": 1, "client_name": 0.8867, "client_id": 0.9933, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 36, "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9513, "doc_accuracy": 0.6933, "usd_per_1000": 0.6184, "latency_ms_p50": 14064, "latency_ms_p90": 23066, "invalid_json_rate": 0, "by_field": { "provider_name": 0.6933, "provider_id": 1, "client_name": 0.6933, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 0.9933, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 0.9933, "governing_law": 0.9933 }, "errors": { "wrong_value": 92, "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9154, "doc_accuracy": 0.4333, "usd_per_1000": 0.17, "latency_ms_p50": 1127, "latency_ms_p90": 1407, "invalid_json_rate": 0, "by_field": { "provider_name": 0.4333, "provider_id": 1, "client_name": 0.4667, "client_id": 1, "contract_type": 1, "price": 1, "currency": 1, "price_period": 1, "effective_date": 1, "end_date": 1, "notice_period_months": 1, "penalty_pct_per_day": 1, "governing_law": 1 }, "errors": { "wrong_value": 165 }, "n_cases": 150, "passes_bar": false, "top_error": "wrong_value", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "pii", "lang": "cs", "level": "all", "bar": 0.95, "rows": [ { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9981, "doc_accuracy": 0.9867, "usd_per_1000": 0.1189, "latency_ms_p50": 2907, "latency_ms_p90": 5834, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9981, "doc_accuracy": 0.9867, "usd_per_1000": 1.3978, "latency_ms_p50": 833, "latency_ms_p90": 1063, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 0.3146, "latency_ms_p50": 4249, "latency_ms_p90": 8531, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.2218, "latency_ms_p50": 2531, "latency_ms_p90": 3300, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.1195, "latency_ms_p50": 1846, "latency_ms_p90": 2421, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9733, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9952, "doc_accuracy": 0.9667, "usd_per_1000": 2.1881, "latency_ms_p50": 2025, "latency_ms_p90": 2981, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3, "hallucinated": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9886, "doc_accuracy": 0.92, "usd_per_1000": 4.0557, "latency_ms_p50": 3178, "latency_ms_p90": 4965, "invalid_json_rate": 0, "by_field": { "person_names": 0.9667, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9733, "birth_dates": 0.98, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 8, "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9875, "doc_accuracy": 0.9195, "usd_per_1000": 0.0926, "latency_ms_p50": 889, "latency_ms_p90": 1251, "invalid_json_rate": 0, "by_field": { "person_names": 0.9396, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9866, "birth_dates": 0.9866, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 8, "missing_field": 5 }, "n_cases": 149, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.981, "doc_accuracy": 0.88, "usd_per_1000": 0.4542, "latency_ms_p50": 793, "latency_ms_p90": 926, "invalid_json_rate": 0, "by_field": { "person_names": 0.9133, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9933, "birth_dates": 0.96, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 14, "missing_field": 5, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9695, "doc_accuracy": 0.8867, "usd_per_1000": 0.4062, "latency_ms_p50": 9622, "latency_ms_p90": 17243, "invalid_json_rate": 0.0067, "by_field": { "person_names": 0.9067, "email_addresses": 0.98, "phone_numbers": 0.9733, "postal_addresses": 0.9667, "birth_dates": 0.98, "national_ids": 0.9933, "bank_accounts": 0.9867 }, "errors": { "wrong_value": 13, "invalid_json": 7, "hallucinated": 6, "missing_field": 6 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "pii", "lang": "cs", "level": "text", "bar": 0.95, "rows": [ { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.9981, "doc_accuracy": 0.9867, "usd_per_1000": 0.1189, "latency_ms_p50": 2907, "latency_ms_p90": 5834, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9981, "doc_accuracy": 0.9867, "usd_per_1000": 1.3978, "latency_ms_p50": 833, "latency_ms_p90": 1063, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 0.3146, "latency_ms_p50": 4249, "latency_ms_p90": 8531, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.2218, "latency_ms_p50": 2531, "latency_ms_p90": 3300, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.1195, "latency_ms_p50": 1846, "latency_ms_p90": 2421, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9733, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9952, "doc_accuracy": 0.9667, "usd_per_1000": 2.1881, "latency_ms_p50": 2025, "latency_ms_p90": 2981, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3, "hallucinated": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9886, "doc_accuracy": 0.92, "usd_per_1000": 4.0557, "latency_ms_p50": 3178, "latency_ms_p90": 4965, "invalid_json_rate": 0, "by_field": { "person_names": 0.9667, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9733, "birth_dates": 0.98, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 8, "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9875, "doc_accuracy": 0.9195, "usd_per_1000": 0.0926, "latency_ms_p50": 889, "latency_ms_p90": 1251, "invalid_json_rate": 0, "by_field": { "person_names": 0.9396, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9866, "birth_dates": 0.9866, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 8, "missing_field": 5 }, "n_cases": 149, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.981, "doc_accuracy": 0.88, "usd_per_1000": 0.4542, "latency_ms_p50": 793, "latency_ms_p90": 926, "invalid_json_rate": 0, "by_field": { "person_names": 0.9133, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9933, "birth_dates": 0.96, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 14, "missing_field": 5, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9695, "doc_accuracy": 0.8867, "usd_per_1000": 0.4062, "latency_ms_p50": 9622, "latency_ms_p90": 17243, "invalid_json_rate": 0.0067, "by_field": { "person_names": 0.9067, "email_addresses": 0.98, "phone_numbers": 0.9733, "postal_addresses": 0.9667, "birth_dates": 0.98, "national_ids": 0.9933, "bank_accounts": 0.9867 }, "errors": { "wrong_value": 13, "invalid_json": 7, "hallucinated": 6, "missing_field": 6 }, "n_cases": 150, "passes_bar": true, "top_error": "wrong_value", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "pii", "lang": "de", "level": "all", "bar": 0.95, "rows": [ { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 1.3835, "latency_ms_p50": 916, "latency_ms_p90": 1233, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 0.1019, "latency_ms_p50": 1876, "latency_ms_p90": 5172, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9933, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9981, "doc_accuracy": 0.9865, "usd_per_1000": 0.3055, "latency_ms_p50": 3669, "latency_ms_p90": 6950, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9865, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2 }, "n_cases": 148, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 4.404, "latency_ms_p50": 3660, "latency_ms_p90": 6811, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 0.9933, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.1196, "latency_ms_p50": 2343, "latency_ms_p90": 3078, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.1268, "latency_ms_p50": 2597, "latency_ms_p90": 3732, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.4543, "latency_ms_p50": 813, "latency_ms_p90": 988, "invalid_json_rate": 0, "by_field": { "person_names": 0.9733, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 3, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.1172, "latency_ms_p50": 2247, "latency_ms_p90": 3094, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9733, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9962, "doc_accuracy": 0.9933, "usd_per_1000": 0.3589, "latency_ms_p50": 12189, "latency_ms_p90": 18612, "invalid_json_rate": 0, "by_field": { "person_names": 0.9933, "email_addresses": 0.9933, "phone_numbers": 0.9933, "postal_addresses": 0.9933, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9908, "doc_accuracy": 0.9353, "usd_per_1000": 0.0899, "latency_ms_p50": 943, "latency_ms_p90": 1272, "invalid_json_rate": 0, "by_field": { "person_names": 0.9496, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9928, "birth_dates": 1, "national_ids": 1, "bank_accounts": 0.9928 }, "errors": { "hallucinated": 7, "missing_field": 1, "wrong_value": 1 }, "n_cases": 139, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "pii", "lang": "de", "level": "text", "bar": 0.95, "rows": [ { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 1, "doc_accuracy": 1, "usd_per_1000": 1.3835, "latency_ms_p50": 916, "latency_ms_p90": 1233, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": {}, "n_cases": 150, "passes_bar": true, "top_error": null, "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 0.1019, "latency_ms_p50": 1876, "latency_ms_p90": 5172, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9933, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9981, "doc_accuracy": 0.9865, "usd_per_1000": 0.3055, "latency_ms_p50": 3669, "latency_ms_p90": 6950, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9865, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2 }, "n_cases": 148, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 4.404, "latency_ms_p50": 3660, "latency_ms_p90": 6811, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 0.9933, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.1196, "latency_ms_p50": 2343, "latency_ms_p90": 3078, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.1268, "latency_ms_p50": 2597, "latency_ms_p90": 3732, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.4543, "latency_ms_p50": 813, "latency_ms_p90": 988, "invalid_json_rate": 0, "by_field": { "person_names": 0.9733, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 1, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 3, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.1172, "latency_ms_p50": 2247, "latency_ms_p90": 3094, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9733, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9962, "doc_accuracy": 0.9933, "usd_per_1000": 0.3589, "latency_ms_p50": 12189, "latency_ms_p90": 18612, "invalid_json_rate": 0, "by_field": { "person_names": 0.9933, "email_addresses": 0.9933, "phone_numbers": 0.9933, "postal_addresses": 0.9933, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9908, "doc_accuracy": 0.9353, "usd_per_1000": 0.0899, "latency_ms_p50": 943, "latency_ms_p90": 1272, "invalid_json_rate": 0, "by_field": { "person_names": 0.9496, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9928, "birth_dates": 1, "national_ids": 1, "bank_accounts": 0.9928 }, "errors": { "hallucinated": 7, "missing_field": 1, "wrong_value": 1 }, "n_cases": 139, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" } ] }, { "task": "pii", "lang": "en", "level": "all", "bar": 0.95, "rows": [ { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 0.0845, "latency_ms_p50": 2396, "latency_ms_p90": 5354, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9933, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 3.5004, "latency_ms_p50": 2455, "latency_ms_p90": 4158, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 1.8639, "latency_ms_p50": 1944, "latency_ms_p90": 2646, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 1.147, "latency_ms_p50": 799, "latency_ms_p90": 1039, "invalid_json_rate": 0, "by_field": { "person_names": 0.9933, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2, "hallucinated": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.0916, "latency_ms_p50": 2598, "latency_ms_p90": 4610, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.2854, "latency_ms_p50": 3776, "latency_ms_p90": 6845, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2, "hallucinated": 1, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.1058, "latency_ms_p50": 1797, "latency_ms_p90": 2673, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 2, "missing_field": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9952, "doc_accuracy": 0.9733, "usd_per_1000": 0.4074, "latency_ms_p50": 776, "latency_ms_p90": 969, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 0.9933, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 0.9867, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4, "hallucinated": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9942, "doc_accuracy": 0.9597, "usd_per_1000": 0.0772, "latency_ms_p50": 834, "latency_ms_p90": 1225, "invalid_json_rate": 0, "by_field": { "person_names": 0.9732, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9866, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 3, "missing_field": 3 }, "n_cases": 149, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9924, "doc_accuracy": 0.9867, "usd_per_1000": 0.2791, "latency_ms_p50": 7481, "latency_ms_p90": 12254, "invalid_json_rate": 0, "by_field": { "person_names": 0.9933, "email_addresses": 0.9933, "phone_numbers": 0.9933, "postal_addresses": 0.9867, "birth_dates": 0.9933, "national_ids": 0.9933, "bank_accounts": 0.9933 }, "errors": { "hallucinated": 5, "wrong_value": 2, "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] }, { "task": "pii", "lang": "en", "level": "text", "bar": 0.95, "rows": [ { "model_id": "google/gemma-4-31b-it", "field_accuracy": 0.999, "doc_accuracy": 0.9933, "usd_per_1000": 0.0845, "latency_ms_p50": 2396, "latency_ms_p90": 5354, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9933, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemma 4 31B", "vendor": "Google" }, { "model_id": "anthropic/claude-sonnet-5", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 3.5004, "latency_ms_p50": 2455, "latency_ms_p90": 4158, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Claude Sonnet 5", "vendor": "Anthropic" }, { "model_id": "google/gemini-3.8-flash", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 1.8639, "latency_ms_p50": 1944, "latency_ms_p90": 2646, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.8 Flash", "vendor": "Google" }, { "model_id": "mistralai/mistral-medium-3-5", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 1.147, "latency_ms_p50": 799, "latency_ms_p90": 1039, "invalid_json_rate": 0, "by_field": { "person_names": 0.9933, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2, "hallucinated": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Mistral Medium 3.5", "vendor": "Mistral" }, { "model_id": "openai/gpt-6-sol", "field_accuracy": 0.9971, "doc_accuracy": 0.98, "usd_per_1000": 2.0916, "latency_ms_p50": 2598, "latency_ms_p90": 4610, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.98, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 3 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "GPT-6 Sol", "vendor": "OpenAI" }, { "model_id": "deepseek/deepseek-v4.1-flash", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.2854, "latency_ms_p50": 3776, "latency_ms_p90": 6845, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 2, "hallucinated": 1, "wrong_value": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "DeepSeek V4.1 Flash", "vendor": "DeepSeek" }, { "model_id": "openai/gpt-6-luna", "field_accuracy": 0.9962, "doc_accuracy": 0.9733, "usd_per_1000": 0.1058, "latency_ms_p50": 1797, "latency_ms_p90": 2673, "invalid_json_rate": 0, "by_field": { "person_names": 0.9867, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 2, "missing_field": 2 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "GPT-6 Luna", "vendor": "OpenAI" }, { "model_id": "google/gemini-3.5-flash-lite", "field_accuracy": 0.9952, "doc_accuracy": 0.9733, "usd_per_1000": 0.4074, "latency_ms_p50": 776, "latency_ms_p90": 969, "invalid_json_rate": 0, "by_field": { "person_names": 1, "email_addresses": 0.9933, "phone_numbers": 1, "postal_addresses": 0.9867, "birth_dates": 0.9867, "national_ids": 1, "bank_accounts": 1 }, "errors": { "missing_field": 4, "hallucinated": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "missing_field", "model_name": "Gemini 3.5 Flash Lite", "vendor": "Google" }, { "model_id": "mistralai/mistral-small-2603", "field_accuracy": 0.9942, "doc_accuracy": 0.9597, "usd_per_1000": 0.0772, "latency_ms_p50": 834, "latency_ms_p90": 1225, "invalid_json_rate": 0, "by_field": { "person_names": 0.9732, "email_addresses": 1, "phone_numbers": 1, "postal_addresses": 0.9866, "birth_dates": 1, "national_ids": 1, "bank_accounts": 1 }, "errors": { "hallucinated": 3, "missing_field": 3 }, "n_cases": 149, "passes_bar": true, "top_error": "hallucinated", "model_name": "Mistral Small 4", "vendor": "Mistral" }, { "model_id": "qwen/qwen3.8-flash", "field_accuracy": 0.9924, "doc_accuracy": 0.9867, "usd_per_1000": 0.2791, "latency_ms_p50": 7481, "latency_ms_p90": 12254, "invalid_json_rate": 0, "by_field": { "person_names": 0.9933, "email_addresses": 0.9933, "phone_numbers": 0.9933, "postal_addresses": 0.9867, "birth_dates": 0.9933, "national_ids": 0.9933, "bank_accounts": 0.9933 }, "errors": { "hallucinated": 5, "wrong_value": 2, "missing_field": 1 }, "n_cases": 150, "passes_bar": true, "top_error": "hallucinated", "model_name": "Qwen3.8 Flash", "vendor": "Qwen" } ] } ] }