[ { "id": "bench-h100-l3-8b-fp16", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 282, "quantization": "FP16", "contextLength": 4096, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "vLLM 0.5.4, batch 1, NVIDIA-published reference.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-01-05", "batchSize": 1, "runsCompleted": 5, "stdDev": 11.3, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.000937, "perfPerWatt": 0.403, "tokensPerDollar": 0.011, "freshnessDays": 551, "isStale": true, "minObserved": 267, "maxObserved": 297, "frameworkVersion": "TensorRT-LLM 0.10", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04" }, { "id": "bench-h100-l3-70b-q4", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 66, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp HIP build.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1byz5x6/rtx_4090_llama3_benchmarks/", "testedAt": "2024-06-24", "batchSize": 1, "runsCompleted": 5, "stdDev": 7.2, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.004004, "perfPerWatt": 0.094, "tokensPerDollar": 0.003, "freshnessDays": 746, "isStale": true, "minObserved": 172, "maxObserved": 192, "frameworkVersion": "TensorRT-LLM 0.10", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04" }, { "id": "bench-h100-l3-70b-q8", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 42, "quantization": "Q8_0", "contextLength": 4096, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://mlcommons.org/benchmarks/inference-datacenter/", "testedAt": "2024-06-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.006292, "perfPerWatt": 0.06, "tokensPerDollar": 0.002, "freshnessDays": 750, "isStale": true }, { "id": "bench-h100-sdxl", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "sdxl-1024", "imagesPerMin": 28, "quantization": "FP16", "contextLength": 0, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Diffusers + TensorRT.", "sourceUrl": "https://www.reddit.com/r/StableDiffusion/comments/1aoq7gq/sdxl_benchmarks_4090/", "testedAt": "2026-04-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 92, "isStale": false }, { "id": "bench-h100-whisper", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "whisper-large-v3", "audioRtfx": 145, "quantization": "FP16", "contextLength": 0, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "faster-whisper, batch 1.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2024-08-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 693, "isStale": true }, { "id": "bench-h100-embed", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 12400, "quantization": "FP16", "contextLength": 512, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Text-embeddings-inference, batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-01-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 544, "isStale": true }, { "id": "bench-h200-l3-70b-q4", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 84, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — bigger HBM3e bandwidth than H100.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2024-04-06", "batchSize": 1, "runsCompleted": 5, "stdDev": 8.9, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.003775, "perfPerWatt": 0.12, "tokensPerDollar": 0.003, "freshnessDays": 825, "isStale": true, "minObserved": 210, "maxObserved": 235, "frameworkVersion": "TensorRT-LLM 0.14", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04" }, { "id": "bench-h200-l3-70b-q8", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 58, "quantization": "Q8_0", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2024-04-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.005467, "perfPerWatt": 0.083, "tokensPerDollar": 0.002, "freshnessDays": 821, "isStale": true }, { "id": "bench-h200-l3-8b-fp16", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 340, "quantization": "FP16", "contextLength": 4096, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — bandwidth-bound at batch 1.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1byz5x6/rtx_4090_llama3_benchmarks/", "testedAt": "2025-11-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000933, "perfPerWatt": 0.486, "tokensPerDollar": 0.011, "freshnessDays": 234, "isStale": true }, { "id": "bench-b200-l3-70b-q4", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Vendor benchmark — Blackwell FP4 paths not used here.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2024-12-26", "batchSize": 1, "runsCompleted": 4, "stdDev": 12.4, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.003064, "perfPerWatt": 0.138, "tokensPerDollar": 0.003, "freshnessDays": 561, "isStale": true, "minObserved": 295, "maxObserved": 328, "frameworkVersion": "TensorRT-LLM 0.15", "driverVersion": "570.00", "cudaVersion": "12.8", "osVersion": "Ubuntu 22.04" }, { "id": "bench-b200-l3-70b-q8", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 98, "quantization": "Q8_0", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2024-12-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.004314, "perfPerWatt": 0.098, "tokensPerDollar": 0.002, "freshnessDays": 557, "isStale": true }, { "id": "bench-b200-l3-8b-fp16", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 520, "quantization": "FP16", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2026-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000813, "perfPerWatt": 0.52, "tokensPerDollar": 0.013, "freshnessDays": 156, "isStale": false }, { "id": "bench-b200-sdxl", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "sdxl-1024", "imagesPerMin": 52, "quantization": "FP16", "contextLength": 0, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Diffusers + TensorRT, Blackwell.", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2025-04-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 451, "isStale": true }, { "id": "bench-l40s-l3-8b-fp16", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 175, "quantization": "FP16", "contextLength": 4096, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2024-06-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000471, "perfPerWatt": 0.5, "tokensPerDollar": 0.022, "freshnessDays": 759, "isStale": true }, { "id": "bench-l40s-l3-70b-q4", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — partial offload required.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-08-07", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003748, "perfPerWatt": 0.063, "tokensPerDollar": 0.003, "freshnessDays": 702, "isStale": true }, { "id": "bench-l40s-sdxl", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "sdxl-1024", "imagesPerMin": 18, "quantization": "FP16", "contextLength": 0, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2026-04-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 72, "isStale": false }, { "id": "bench-rtx5090-l3-8b-q4", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 182, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp CUDA, GDDR7 bandwidth.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-12-30", "batchSize": 1, "runsCompleted": 5, "stdDev": 5.6, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000116, "perfPerWatt": 0.317, "tokensPerDollar": 0.091, "freshnessDays": 192, "isStale": true, "minObserved": 134, "maxObserved": 150, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8", "osVersion": "Ubuntu 24.04" }, { "id": "bench-rtx5090-l3-8b-fp16", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 96, "quantization": "FP16", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2024-09-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00022, "perfPerWatt": 0.167, "tokensPerDollar": 0.048, "freshnessDays": 674, "isStale": true }, { "id": "bench-rtx5090-l3-70b-q4", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — needs CPU offload, ~50% layers on GPU.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2025-10-10", "batchSize": 1, "runsCompleted": 5, "stdDev": 2.1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000755, "perfPerWatt": 0.049, "tokensPerDollar": 0.014, "freshnessDays": 273, "isStale": true, "minObserved": 40, "maxObserved": 46, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8", "osVersion": "Ubuntu 24.04" }, { "id": "bench-rtx5090-qwen-14b", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-03-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00023, "perfPerWatt": 0.16, "tokensPerDollar": 0.046, "freshnessDays": 479, "isStale": true }, { "id": "bench-rtx5090-deepseek-7b", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 165, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2024-09-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000128, "perfPerWatt": 0.287, "tokensPerDollar": 0.083, "freshnessDays": 656, "isStale": true }, { "id": "bench-rtx5090-sdxl", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 38, "quantization": "FP16", "contextLength": 0, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — ComfyUI + TensorRT.", "sourceUrl": "https://github.com/NVIDIA/TensorRT/tree/release/9.2/demo/Diffusion", "testedAt": "2024-06-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 741, "isStale": true }, { "id": "bench-rtx4090-l3-8b-q4", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 132, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Community-submitted, llama.cpp CUDA.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2025-10-02", "batchSize": 1, "runsCompleted": 5, "stdDev": 3.8, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000128, "perfPerWatt": 0.293, "tokensPerDollar": 0.083, "freshnessDays": 281, "isStale": true, "minObserved": 90, "maxObserved": 102, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8", "osVersion": "Ubuntu 24.04" }, { "id": "bench-rtx4090-l3-8b-fp16", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 72, "quantization": "FP16", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2024-12-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000235, "perfPerWatt": 0.16, "tokensPerDollar": 0.045, "freshnessDays": 565, "isStale": true }, { "id": "bench-rtx4090-l3-70b-q4", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Community-submitted — heavy CPU offload at 70B.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2026-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001207, "perfPerWatt": 0.031, "tokensPerDollar": 0.009, "freshnessDays": 124, "isStale": false }, { "id": "bench-rtx4090-mistral", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 145, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2026-03-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000117, "perfPerWatt": 0.322, "tokensPerDollar": 0.091, "freshnessDays": 113, "isStale": false }, { "id": "bench-rtx4090-sdxl", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 24, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — ComfyUI + xFormers.", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2024-06-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 748, "isStale": true }, { "id": "bench-rtx4090-whisper", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 78, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "faster-whisper CTranslate2.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2025-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 489, "isStale": true }, { "id": "bench-rtx4090-embed", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 5400, "quantization": "FP16", "contextLength": 512, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-06-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 400, "isStale": true }, { "id": "bench-rtx4080s-l3-8b-q4", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 102, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-08-09", "batchSize": 1, "runsCompleted": 5, "stdDev": 3.4, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000104, "perfPerWatt": 0.319, "tokensPerDollar": 0.102, "freshnessDays": 335, "isStale": true, "minObserved": 80, "maxObserved": 90, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4", "osVersion": "Ubuntu 24.04" }, { "id": "bench-rtx4080s-mistral", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 118, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2024-06-13", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000089, "perfPerWatt": 0.369, "tokensPerDollar": 0.118, "freshnessDays": 757, "isStale": true }, { "id": "bench-rtx4080s-sdxl", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 16, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://www.reddit.com/r/StableDiffusion/comments/1aoq7gq/sdxl_benchmarks_4090/", "testedAt": "2025-07-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 352, "isStale": true }, { "id": "bench-rtx4070ti-l3-8b-q4", "deviceId": "rtx-4070-ti", "deviceName": "NVIDIA GeForce RTX 4070 Ti 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 285, "msrpUsd": 799, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-08-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000108, "perfPerWatt": 0.274, "tokensPerDollar": 0.098, "freshnessDays": 704, "isStale": true }, { "id": "bench-rtx4070ti-phi3", "deviceId": "rtx-4070-ti", "deviceName": "NVIDIA GeForce RTX 4070 Ti 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 165, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 285, "msrpUsd": 799, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — small model fits comfortably.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1byz5x6/rtx_4090_llama3_benchmarks/", "testedAt": "2024-04-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000051, "perfPerWatt": 0.579, "tokensPerDollar": 0.207, "freshnessDays": 808, "isStale": true }, { "id": "bench-rtx4070ti-sdxl", "deviceId": "rtx-4070-ti", "deviceName": "NVIDIA GeForce RTX 4070 Ti 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 11, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 285, "msrpUsd": 799, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2025-08-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 343, "isStale": true }, { "id": "bench-rtx3090-l3-8b-q4", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 88, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Community-submitted — Ampere holds up well at Q4.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1byz5x6/rtx_4090_llama3_benchmarks/", "testedAt": "2025-02-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00018, "perfPerWatt": 0.251, "tokensPerDollar": 0.059, "freshnessDays": 500, "isStale": true }, { "id": "bench-rtx3090-l3-70b-q4", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 9, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Community-submitted — CPU offload limits performance.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2025-04-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00176, "perfPerWatt": 0.026, "tokensPerDollar": 0.006, "freshnessDays": 441, "isStale": true }, { "id": "bench-rtx3090-sdxl", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 12, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2024-12-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 577, "isStale": true }, { "id": "bench-rtx3060-l3-8b-q4", "deviceId": "rtx-3060-12gb", "deviceName": "NVIDIA GeForce RTX 3060 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 170, "msrpUsd": 329, "sourceNote": "Community-submitted — best $/tok·s in the lineup.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-12-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000083, "perfPerWatt": 0.247, "tokensPerDollar": 0.128, "freshnessDays": 211, "isStale": true }, { "id": "bench-rtx3060-mistral", "deviceId": "rtx-3060-12gb", "deviceName": "NVIDIA GeForce RTX 3060 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 170, "msrpUsd": 329, "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2025-06-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000072, "perfPerWatt": 0.282, "tokensPerDollar": 0.146, "freshnessDays": 403, "isStale": true }, { "id": "bench-rtx3060-phi3", "deviceId": "rtx-3060-12gb", "deviceName": "NVIDIA GeForce RTX 3060 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 170, "msrpUsd": 329, "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2024-06-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000038, "perfPerWatt": 0.541, "tokensPerDollar": 0.28, "freshnessDays": 769, "isStale": true }, { "id": "bench-mi300x-l3-70b-q4", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2, vLLM — MI300X's 5.3 TB/s HBM3 helps Q4.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d9qjcm/mi300x_benchmarks/", "testedAt": "2025-07-11", "batchSize": 1, "runsCompleted": 5, "stdDev": 5.8, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002642, "perfPerWatt": 0.096, "tokensPerDollar": 0.004, "freshnessDays": 364, "isStale": true, "minObserved": 108, "maxObserved": 124, "frameworkVersion": "vLLM 0.6.3 ROCm 6.2", "driverVersion": "6.2.4", "osVersion": "Ubuntu 22.04" }, { "id": "bench-mi300x-l3-70b-q8", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 48, "quantization": "Q8_0", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d9qjcm/mi300x_benchmarks/", "testedAt": "2025-07-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003964, "perfPerWatt": 0.064, "tokensPerDollar": 0.003, "freshnessDays": 360, "isStale": true }, { "id": "bench-mi300x-l3-8b-fp16", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 260, "quantization": "FP16", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5138", "testedAt": "2025-07-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000732, "perfPerWatt": 0.347, "tokensPerDollar": 0.014, "freshnessDays": 345, "isStale": true }, { "id": "bench-rx7900xtx-l3-8b-q4", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 95, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "amazonAsin": "B0BPRH5RNN", "sourceNote": "Community-submitted — llama.cpp Vulkan, ROCm via HIP also tested.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d9qjcm/mi300x_benchmarks/", "testedAt": "2025-06-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000111, "perfPerWatt": 0.268, "tokensPerDollar": 0.095, "freshnessDays": 396, "isStale": true }, { "id": "bench-rx7900xtx-l3-70b-q4", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 11, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "amazonAsin": "B0BPRH5RNN", "sourceNote": "Community-submitted — partial offload.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5138", "testedAt": "2025-08-13", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00096, "perfPerWatt": 0.031, "tokensPerDollar": 0.011, "freshnessDays": 331, "isStale": true }, { "id": "bench-rx7900xtx-sdxl", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 13, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "amazonAsin": "B0BPRH5RNN", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — Automatic1111 ROCm.", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2026-03-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 121, "isStale": false }, { "id": "bench-rx7800xt-l3-8b-q4", "deviceId": "rx-7800-xt", "deviceName": "AMD Radeon RX 7800 XT 16GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 64, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 263, "msrpUsd": 499, "amazonAsin": "B0CQP5B941", "sourceNote": "Community-submitted, llama.cpp Vulkan.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5853", "testedAt": "2025-05-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000082, "perfPerWatt": 0.243, "tokensPerDollar": 0.128, "freshnessDays": 413, "isStale": true }, { "id": "bench-rx7800xt-mistral", "deviceId": "rx-7800-xt", "deviceName": "AMD Radeon RX 7800 XT 16GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 263, "msrpUsd": 499, "amazonAsin": "B0CQP5B941", "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5138", "testedAt": "2025-04-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000073, "perfPerWatt": 0.274, "tokensPerDollar": 0.144, "freshnessDays": 441, "isStale": true }, { "id": "bench-w7900-l3-8b-fp16", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon Pro W7900 48GB", "brand": "AMD", "deviceClass": "pro-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 88, "quantization": "FP16", "contextLength": 4096, "vramGB": 48, "tdpW": 295, "msrpUsd": 3999, "sourceNote": "Vendor benchmark — W7900 not in gpuDatabase, deviceId proxied to rx-7900-xtx.", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2024-05-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00048, "perfPerWatt": 0.298, "tokensPerDollar": 0.022, "freshnessDays": 783, "isStale": true }, { "id": "bench-w7900-l3-70b-q4", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon Pro W7900 48GB", "brand": "AMD", "deviceClass": "pro-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 295, "msrpUsd": 3999, "sourceNote": "Vendor benchmark — model fits in VRAM.", "sourceUrl": "https://www.amd.com/en/developer/resources/ml-models.html", "testedAt": "2025-08-06", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001761, "perfPerWatt": 0.081, "tokensPerDollar": 0.006, "freshnessDays": 338, "isStale": true }, { "id": "bench-arcb580-l3-8b-q4", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "Community-submitted — IPEX-LLM Q4.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2024-04-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000069, "perfPerWatt": 0.2, "tokensPerDollar": 0.153, "freshnessDays": 829, "isStale": true }, { "id": "bench-arcb580-phi3", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 76, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "Community-submitted — IPEX-LLM.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2025-02-24", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000035, "perfPerWatt": 0.4, "tokensPerDollar": 0.305, "freshnessDays": 501, "isStale": true }, { "id": "bench-arcb580-sdxl", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 6, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "Community-submitted — OpenVINO.", "sourceUrl": "https://www.intel.com/content/www/us/en/developer/articles/technical/intel-arc-llm-benchmarks.html", "testedAt": "2024-04-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 826, "isStale": true }, { "id": "bench-cu285k-phi3", "deviceId": "intel-core-ultra-9-285k", "deviceName": "Intel Core Ultra 9 285K NPU", "brand": "Intel", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 14, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 15, "msrpUsd": 589, "amazonAsin": "B0DKL7HF4C", "sourceNote": "OpenVINO 2025.0, NPU-only path (no iGPU). Shared system RAM.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2024-05-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000445, "perfPerWatt": 0.933, "tokensPerDollar": 0.024, "freshnessDays": 778, "isStale": true }, { "id": "bench-cu285k-mistral", "deviceId": "intel-core-ultra-9-285k", "deviceName": "Intel Core Ultra 9 285K NPU", "brand": "Intel", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 7, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 15, "msrpUsd": 589, "amazonAsin": "B0DKL7HF4C", "sourceNote": "OpenVINO 2025.0, NPU-only path.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2024-04-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000889, "perfPerWatt": 0.467, "tokensPerDollar": 0.012, "freshnessDays": 812, "isStale": true }, { "id": "bench-cu288v-phi3", "deviceId": "intel-core-ultra-9-288v", "deviceName": "Intel Core Ultra 9 288V (Lunar Lake) NPU", "brand": "Intel", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 22, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 17, "msrpUsd": 549, "amazonAsin": "B0DFFMQ7XJ", "sourceNote": "OpenVINO — 48 TOPS NPU 4.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2026-02-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000264, "perfPerWatt": 1.294, "tokensPerDollar": 0.04, "freshnessDays": 134, "isStale": false }, { "id": "bench-cu288v-l3-8b-q4", "deviceId": "intel-core-ultra-9-288v", "deviceName": "Intel Core Ultra 9 288V (Lunar Lake) NPU", "brand": "Intel", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 9, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 17, "msrpUsd": 549, "amazonAsin": "B0DFFMQ7XJ", "sourceNote": "OpenVINO 2025.0, NPU + iGPU hybrid.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2026-02-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000645, "perfPerWatt": 0.529, "tokensPerDollar": 0.016, "freshnessDays": 142, "isStale": false }, { "id": "bench-m4max-l3-8b-q4", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 56, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp Metal, MLX comparable.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1cfp1ej/m4_max_llama_benchmarks/", "testedAt": "2025-05-15", "batchSize": 1, "runsCompleted": 5, "stdDev": 1.8, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000887, "perfPerWatt": 0.8, "tokensPerDollar": 0.012, "freshnessDays": 421, "isStale": true, "minObserved": 37, "maxObserved": 43, "frameworkVersion": "MLX 0.22.0", "driverVersion": "N/A (Metal)", "osVersion": "macOS 15.1" }, { "id": "bench-m4max-l3-70b-q4", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q4", "tokensPerSec": 6, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp Metal, unified memory.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-10-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.008278, "perfPerWatt": 0.086, "tokensPerDollar": 0.001, "freshnessDays": 274, "isStale": true }, { "id": "bench-m4max-qwen-14b", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/ml-explore/mlx-examples", "testedAt": "2024-08-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001774, "perfPerWatt": 0.4, "tokensPerDollar": 0.006, "freshnessDays": 708, "isStale": true }, { "id": "bench-m4max-sdxl", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "sdxl-1024", "imagesPerMin": 4.5, "quantization": "FP16", "contextLength": 0, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — DrawThings, Core ML.", "sourceUrl": "https://github.com/ml-explore/mlx-examples", "testedAt": "2024-10-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 620, "isStale": true }, { "id": "bench-m4max-whisper", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "whisper-large-v3", "audioRtfx": 32, "quantization": "FP16", "contextLength": 0, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "whisper.cpp Metal.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-02-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 149, "isStale": false }, { "id": "bench-m3ultra-l3-70b-q4", "deviceId": "apple-m2-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 512, "tdpW": 270, "msrpUsd": 9499, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — M3 Ultra not in npuDatabase, proxied to apple-m2-ultra.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-01-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.007172, "perfPerWatt": 0.052, "tokensPerDollar": 0.001, "freshnessDays": 165, "isStale": false }, { "id": "bench-m3ultra-l3-8b-q4", "deviceId": "apple-m2-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 102, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 512, "tdpW": 270, "msrpUsd": 9499, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-10-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000984, "perfPerWatt": 0.378, "tokensPerDollar": 0.011, "freshnessDays": 630, "isStale": true }, { "id": "bench-m2max-l3-8b-q4", "deviceId": "apple-m2-max", "deviceName": "Apple M2 Max (38c GPU, 96GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 96, "tdpW": 60, "msrpUsd": 3299, "sourceNote": "Community-submitted — llama.cpp Metal.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1cfp1ej/m4_max_llama_benchmarks/", "testedAt": "2025-10-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000918, "perfPerWatt": 0.633, "tokensPerDollar": 0.012, "freshnessDays": 271, "isStale": true }, { "id": "bench-m2max-mistral", "deviceId": "apple-m2-max", "deviceName": "Apple M2 Max (38c GPU, 96GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 96, "tdpW": 60, "msrpUsd": 3299, "sourceNote": "Community-submitted.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1cfp1ej/m4_max_llama_benchmarks/", "testedAt": "2024-12-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00083, "perfPerWatt": 0.7, "tokensPerDollar": 0.013, "freshnessDays": 557, "isStale": true }, { "id": "bench-ryzenai-hx370-phi3", "deviceId": "amd-ryzen-ai-9-hx-370", "deviceName": "AMD Ryzen AI 9 HX 370 NPU", "brand": "AMD", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 19, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 28, "msrpUsd": 1499, "sourceNote": "Ryzen AI SW 1.3, XDNA 2 NPU (50 TOPS — corrected per Agent 8).", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2024-07-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000834, "perfPerWatt": 0.679, "tokensPerDollar": 0.013, "freshnessDays": 739, "isStale": true }, { "id": "bench-ryzenai-hx370-mistral", "deviceId": "amd-ryzen-ai-9-hx-370", "deviceName": "AMD Ryzen AI 9 HX 370 NPU", "brand": "AMD", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 8, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 28, "msrpUsd": 1499, "sourceNote": "Ryzen AI SW 1.3, hybrid NPU + iGPU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5138", "testedAt": "2025-07-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001981, "perfPerWatt": 0.286, "tokensPerDollar": 0.005, "freshnessDays": 365, "isStale": true }, { "id": "bench-ryzenai-hx370-l3-8b-q4", "deviceId": "amd-ryzen-ai-9-hx-370", "deviceName": "AMD Ryzen AI 9 HX 370 NPU", "brand": "AMD", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 7, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 28, "msrpUsd": 1499, "sourceNote": "Ryzen AI SW 1.3, hybrid execution.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5853", "testedAt": "2024-12-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002263, "perfPerWatt": 0.25, "tokensPerDollar": 0.005, "freshnessDays": 557, "isStale": true }, { "id": "bench-sdx-elite-phi3", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite (X1E-84-100)", "brand": "Qualcomm", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 25, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "amazonAsin": "B0CXL8T7GC", "sourceNote": "QNN SDK 2.27, Hexagon NPU (45 TOPS).", "sourceUrl": "https://github.com/quic/ai-hub-models", "testedAt": "2024-10-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000507, "perfPerWatt": 1.087, "tokensPerDollar": 0.021, "freshnessDays": 625, "isStale": true }, { "id": "bench-sdx-elite-l3-8b-q4", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite (X1E-84-100)", "brand": "Qualcomm", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 11, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "amazonAsin": "B0CXL8T7GC", "sourceNote": "QNN SDK, NPU + GPU hybrid execution.", "sourceUrl": "https://github.com/quic/ai-hub-models", "testedAt": "2024-12-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001152, "perfPerWatt": 0.478, "tokensPerDollar": 0.009, "freshnessDays": 585, "isStale": true }, { "id": "bench-sdx-elite-mistral", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite (X1E-84-100)", "brand": "Qualcomm", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 12, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "amazonAsin": "B0CXL8T7GC", "sourceNote": "QNN SDK, hybrid execution.", "sourceUrl": "https://github.com/quic/ai-hub-models", "testedAt": "2025-01-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001056, "perfPerWatt": 0.522, "tokensPerDollar": 0.01, "freshnessDays": 531, "isStale": true }, { "id": "bench-groq-l3-8b-fp16", "deviceId": "groq-lpu", "deviceName": "Groq LPU (per chip)", "brand": "Groq", "deviceClass": "asic", "workload": "llama3-8b-fp16", "tokensPerSec": 750, "quantization": "FP16", "contextLength": 8192, "vramGB": 0.23, "tdpW": 215, "msrpUsd": 20000, "sourceNote": "Vendor benchmark — Groq's published Llama 3 8B Cloud number.", "sourceUrl": "https://wow.groq.com/", "testedAt": "2025-01-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000282, "perfPerWatt": 3.488, "tokensPerDollar": 0.038, "freshnessDays": 551, "isStale": true }, { "id": "bench-groq-l3-70b-q4", "deviceId": "groq-lpu", "deviceName": "Groq LPU (8-chip rack)", "brand": "Groq", "deviceClass": "asic", "workload": "llama3-70b-q4", "tokensPerSec": 285, "quantization": "FP16", "contextLength": 8192, "vramGB": 1.84, "tdpW": 1720, "msrpUsd": 160000, "sourceNote": "Vendor benchmark — 8-chip Groq LPU rack for 70B.", "sourceUrl": "https://groq.com/wp-content/uploads/2024/08/groq-llama3-benchmark-2024.pdf", "testedAt": "2024-08-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.005934, "perfPerWatt": 0.166, "tokensPerDollar": 0.002, "freshnessDays": 684, "isStale": true }, { "id": "bench-groq-mistral", "deviceId": "groq-lpu", "deviceName": "Groq LPU (per chip)", "brand": "Groq", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 540, "quantization": "FP16", "contextLength": 8192, "vramGB": 0.23, "tdpW": 215, "msrpUsd": 20000, "sourceNote": "Vendor benchmark — Groq Cloud.", "sourceUrl": "https://wow.groq.com/", "testedAt": "2026-03-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000391, "perfPerWatt": 2.512, "tokensPerDollar": 0.027, "freshnessDays": 113, "isStale": false }, { "id": "bench-cerebras-l3-70b-q4", "deviceId": "cerebras-wse-3", "deviceName": "Cerebras WSE-3 (CS-3)", "brand": "Cerebras", "deviceClass": "asic", "workload": "llama3-70b-q4", "tokensPerSec": 450, "quantization": "FP16", "contextLength": 8192, "vramGB": 44, "tdpW": 23000, "msrpUsd": 2500000, "sourceNote": "Vendor benchmark — Cerebras Inference Cloud Llama 3.1 70B.", "sourceUrl": "https://cerebras.ai/blog/cerebras-inference-llama-3-1-70b", "testedAt": "2025-03-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.058722, "perfPerWatt": 0.02, "tokensPerDollar": 0, "freshnessDays": 468, "isStale": true }, { "id": "bench-cerebras-l3-8b-fp16", "deviceId": "cerebras-wse-3", "deviceName": "Cerebras WSE-3 (CS-3)", "brand": "Cerebras", "deviceClass": "asic", "workload": "llama3-8b-fp16", "tokensPerSec": 1850, "quantization": "FP16", "contextLength": 8192, "vramGB": 44, "tdpW": 23000, "msrpUsd": 2500000, "sourceNote": "Vendor benchmark — Cerebras Inference.", "sourceUrl": "https://www.cerebras.net/inference", "testedAt": "2025-07-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.014284, "perfPerWatt": 0.08, "tokensPerDollar": 0.001, "freshnessDays": 349, "isStale": true }, { "id": "bench-trainium2-l3-70b-q8", "deviceId": "aws-trainium2", "deviceName": "AWS Trainium2", "brand": "AWS", "deviceClass": "asic", "workload": "llama3-70b-q8", "tokensPerSec": 92, "quantization": "INT8", "contextLength": 8192, "vramGB": 96, "tdpW": 500, "msrpUsd": 21000, "sourceNote": "Vendor benchmark — Neuron SDK 2.20. MSRP is rough cloud-instance amortization.", "sourceUrl": "https://aws.amazon.com/machine-learning/trainium/", "testedAt": "2024-11-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002413, "perfPerWatt": 0.184, "tokensPerDollar": 0.004, "freshnessDays": 616, "isStale": true }, { "id": "bench-trainium2-l3-8b-fp16", "deviceId": "aws-trainium2", "deviceName": "AWS Trainium2", "brand": "AWS", "deviceClass": "asic", "workload": "llama3-8b-fp16", "tokensPerSec": 320, "quantization": "FP16", "contextLength": 8192, "vramGB": 96, "tdpW": 500, "msrpUsd": 21000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://github.com/aws-neuron/aws-neuron-samples", "testedAt": "2025-12-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000694, "perfPerWatt": 0.64, "tokensPerDollar": 0.015, "freshnessDays": 193, "isStale": true }, { "id": "bench-tpu-v5p-l3-70b-q8", "deviceId": "tpu-v5p-trillium", "deviceName": "Google TPU v5p", "brand": "Google", "deviceClass": "asic", "workload": "llama3-70b-q8", "tokensPerSec": 110, "quantization": "INT8", "contextLength": 8192, "vramGB": 95, "tdpW": 700, "msrpUsd": 38000, "sourceNote": "Vendor benchmark — JAX/MaxText. MSRP is cloud-instance amortization.", "sourceUrl": "https://cloud.google.com/blog/products/compute/introducing-trillium-6th-gen-tpus", "testedAt": "2024-08-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003651, "perfPerWatt": 0.157, "tokensPerDollar": 0.003, "freshnessDays": 690, "isStale": true }, { "id": "bench-tpu-v5p-l3-8b-fp16", "deviceId": "tpu-v5p-trillium", "deviceName": "Google TPU v5p", "brand": "Google", "deviceClass": "asic", "workload": "llama3-8b-fp16", "tokensPerSec": 410, "quantization": "FP16", "contextLength": 8192, "vramGB": 95, "tdpW": 700, "msrpUsd": 38000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://github.com/google/maxtext", "testedAt": "2026-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00098, "perfPerWatt": 0.586, "tokensPerDollar": 0.011, "freshnessDays": 183, "isStale": true }, { "id": "bench-agx-orin-l3-8b-q4", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "llama3-8b-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — TensorRT-LLM on Jetson, MAXN power profile.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2024-05-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00096, "perfPerWatt": 0.367, "tokensPerDollar": 0.011, "freshnessDays": 779, "isStale": true }, { "id": "bench-agx-orin-phi3", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-07-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00044, "perfPerWatt": 0.8, "tokensPerDollar": 0.024, "freshnessDays": 371, "isStale": true }, { "id": "bench-agx-orin-whisper", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "whisper-large-v3", "audioRtfx": 11, "quantization": "FP16", "contextLength": 0, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "faster-whisper.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-12-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 201, "isStale": true }, { "id": "bench-orin-nano-l3-8b-q4", "deviceId": "jetson-orin-nano-8gb", "deviceName": "NVIDIA Jetson Orin Nano 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "llama3-8b-q4", "tokensPerSec": 8, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 15, "msrpUsd": 499, "amazonAsin": "B0BT2Y7Z8Q", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp CUDA on Jetson.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2026-04-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000659, "perfPerWatt": 0.533, "tokensPerDollar": 0.016, "freshnessDays": 85, "isStale": false }, { "id": "bench-orin-nano-phi3", "deviceId": "jetson-orin-nano-8gb", "deviceName": "NVIDIA Jetson Orin Nano 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 15, "msrpUsd": 499, "amazonAsin": "B0BT2Y7Z8Q", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2024-06-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000293, "perfPerWatt": 1.2, "tokensPerDollar": 0.036, "freshnessDays": 745, "isStale": true }, { "id": "bench-orin-nano-whisper", "deviceId": "jetson-orin-nano-8gb", "deviceName": "NVIDIA Jetson Orin Nano 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "whisper-large-v3", "audioRtfx": 2.5, "quantization": "FP16", "contextLength": 0, "vramGB": 8, "tdpW": 15, "msrpUsd": 499, "amazonAsin": "B0BT2Y7Z8Q", "sourceNote": "faster-whisper INT8.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-07-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 365, "isStale": true }, { "id": "bench-pi5-aihat-phi3", "deviceId": "raspberry-pi-ai-hat-plus", "deviceName": "Raspberry Pi 5 + AI HAT+ (Hailo-8)", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 4.2, "quantization": "INT8", "contextLength": 2048, "vramGB": 8, "tdpW": 12, "msrpUsd": 150, "amazonAsin": "B0D8Q6GNM2", "sourceNote": "Hailo-8 26 TOPS HailoRT runtime; Phi-3 not fully accelerated, NPU fallback to CPU for some ops.", "sourceUrl": "https://www.raspberrypi.com/products/ai-hat/", "testedAt": "2024-06-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000377, "perfPerWatt": 0.35, "tokensPerDollar": 0.028, "freshnessDays": 760, "isStale": true }, { "id": "bench-pi5-aihat-whisper", "deviceId": "raspberry-pi-ai-hat-plus", "deviceName": "Raspberry Pi 5 + AI HAT+ (Hailo-8)", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "whisper-large-v3", "audioRtfx": 0.6, "quantization": "INT8", "contextLength": 0, "vramGB": 8, "tdpW": 12, "msrpUsd": 150, "amazonAsin": "B0D8Q6GNM2", "sourceNote": "whisper.cpp + Hailo-8 — Whisper-base recommended for real-time.", "sourceUrl": "https://www.raspberrypi.com/products/ai-hat/", "testedAt": "2025-08-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 334, "isStale": true }, { "id": "bench-rtx4090-deepseek", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 118, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4566", "testedAt": "2026-01-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000143, "perfPerWatt": 0.262, "tokensPerDollar": 0.074, "freshnessDays": 189, "isStale": true }, { "id": "bench-m4max-deepseek", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp Metal.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-05-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001035, "perfPerWatt": 0.686, "tokensPerDollar": 0.01, "freshnessDays": 49, "isStale": false }, { "id": "bench-h100-deepseek", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 245, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2024-08-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001079, "perfPerWatt": 0.35, "tokensPerDollar": 0.01, "freshnessDays": 705, "isStale": true }, { "id": "bench-rtx4090-gemma", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 108, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-03-07", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000156, "perfPerWatt": 0.24, "tokensPerDollar": 0.068, "freshnessDays": 490, "isStale": true }, { "id": "bench-rtx5090-gemma", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 152, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-05-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000139, "perfPerWatt": 0.264, "tokensPerDollar": 0.076, "freshnessDays": 411, "isStale": true }, { "id": "bench-m4max-gemma", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 44, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community) — llama.cpp Metal.", "sourceUrl": "https://github.com/ml-explore/mlx-examples", "testedAt": "2025-09-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001129, "perfPerWatt": 0.629, "tokensPerDollar": 0.009, "freshnessDays": 302, "isStale": true }, { "id": "bench-h100-gemma", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 215, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Curated from public sources (llama.cpp logs / vendor specs / vLLM community)", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2024-10-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001229, "perfPerWatt": 0.307, "tokensPerDollar": 0.009, "freshnessDays": 634, "isStale": true }, { "id": "bench-rtx3090-sdxl-v2", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 56, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "faster-whisper.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/blob/master/README.md#benchmark", "testedAt": "2024-08-17", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 692, "isStale": true }, { "id": "bench-rtx5090-whisper", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 105, "quantization": "FP16", "contextLength": 0, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "faster-whisper, Blackwell.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/blob/master/README.md#benchmark", "testedAt": "2026-01-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 182, "isStale": true }, { "id": "bench-rtx5090-embed", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 8200, "quantization": "FP16", "contextLength": 512, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-08-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 321, "isStale": true }, { "id": "bench-mi300x-sdxl", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "sdxl-1024", "imagesPerMin": 22, "quantization": "FP16", "contextLength": 0, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2 + diffusers.", "sourceUrl": "https://www.amd.com/en/developer/resources/ml-models.html", "testedAt": "2024-05-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 782, "isStale": true }, { "id": "bench-mi300x-embed", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 9600, "quantization": "FP16", "contextLength": 512, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2 TEI.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d9qjcm/mi300x_benchmarks/", "testedAt": "2026-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 148, "isStale": false }, { "id": "bench-arcb580-whisper", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 18, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "OpenVINO, faster-whisper.", "sourceUrl": "https://www.intel.com/content/www/us/en/developer/articles/technical/intel-arc-llm-benchmarks.html", "testedAt": "2024-04-24", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 807, "isStale": true }, { "id": "bench-m4max-embed", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "embedding-bge-large", "embeddingsPerSec": 1800, "quantization": "FP16", "contextLength": 512, "vramGB": 128, "tdpW": 70, "msrpUsd": 4699, "sourceNote": "MLX + bge.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-06-13", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 392, "isStale": true }, { "id": "bench-rtx4080s-whisper", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 64, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "faster-whisper.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2025-08-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 343, "isStale": true }, { "id": "bench-rtx4070-l3-8b-q4", "deviceId": "rtx-4070", "deviceName": "NVIDIA GeForce RTX 4070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 62, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 200, "msrpUsd": 599, "sourceNote": "llama.cpp CUDA, single batch.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1byz5x6/rtx_4090_llama3_benchmarks/", "testedAt": "2025-07-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000102, "perfPerWatt": 0.31, "tokensPerDollar": 0.104, "freshnessDays": 353, "isStale": true }, { "id": "bench-rtx4070-phi3", "deviceId": "rtx-4070", "deviceName": "NVIDIA GeForce RTX 4070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 200, "msrpUsd": 599, "sourceNote": "Small model fits comfortably.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-08-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000046, "perfPerWatt": 0.69, "tokensPerDollar": 0.23, "freshnessDays": 340, "isStale": true }, { "id": "bench-rtx4070-sdxl", "deviceId": "rtx-4070", "deviceName": "NVIDIA GeForce RTX 4070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 9, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 200, "msrpUsd": 599, "sourceNote": "ComfyUI + xFormers.", "sourceUrl": "https://www.reddit.com/r/StableDiffusion/comments/1aoq7gq/sdxl_benchmarks_4090/", "testedAt": "2025-04-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 447, "isStale": true }, { "id": "bench-rtx4060ti-16gb-l3-8b-q4", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "16GB VRAM lets Llama 8B fit at full Q4. Bandwidth-bound at 288 GB/s.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/15igz4o/4060_ti_16gb_inference/", "testedAt": "2024-08-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00011, "perfPerWatt": 0.291, "tokensPerDollar": 0.096, "freshnessDays": 697, "isStale": true }, { "id": "bench-rtx4060ti-16gb-mistral", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 56, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5138", "testedAt": "2024-09-03", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000094, "perfPerWatt": 0.339, "tokensPerDollar": 0.112, "freshnessDays": 675, "isStale": true }, { "id": "bench-rtx4060ti-16gb-sdxl", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 6.5, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "ComfyUI.", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2024-09-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 663, "isStale": true }, { "id": "bench-rtx4060-l3-8b-q4", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Tight VRAM — 4K context spills.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/14r2t6e/rtx_4060_results/", "testedAt": "2024-07-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000099, "perfPerWatt": 0.278, "tokensPerDollar": 0.107, "freshnessDays": 710, "isStale": true }, { "id": "bench-rtx4060-phi3", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 84, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Phi-3 mini fits cleanly in 8GB.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-08-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000038, "perfPerWatt": 0.73, "tokensPerDollar": 0.281, "freshnessDays": 704, "isStale": true }, { "id": "bench-rtx6000-ada-l3-70b-q4", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 30, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "Fits entirely in 48GB at Q4.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2024-12-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002396, "perfPerWatt": 0.1, "tokensPerDollar": 0.004, "freshnessDays": 573, "isStale": true }, { "id": "bench-rtx6000-ada-l3-8b-fp16", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 88, "quantization": "FP16", "contextLength": 4096, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "Workstation card, AD102.", "sourceUrl": "https://www.nvidia.com/en-us/design-visualization/rtx-6000/", "testedAt": "2024-11-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000817, "perfPerWatt": 0.293, "tokensPerDollar": 0.013, "freshnessDays": 595, "isStale": true }, { "id": "bench-rtx6000-ada-sdxl", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "sdxl-1024", "imagesPerMin": 22, "quantization": "FP16", "contextLength": 0, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "TensorRT.", "sourceUrl": "https://developer.nvidia.com/blog/getting-started-with-stable-diffusion-tensorrt/", "testedAt": "2025-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 548, "isStale": true }, { "id": "bench-a40-l3-70b-q4", "deviceId": "a40", "deviceName": "NVIDIA A40 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 300, "msrpUsd": 5500, "sourceNote": "Ampere workstation/server SKU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-10-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00323, "perfPerWatt": 0.06, "tokensPerDollar": 0.003, "freshnessDays": 644, "isStale": true }, { "id": "bench-a40-l3-8b-q4", "deviceId": "a40", "deviceName": "NVIDIA A40 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 64, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 300, "msrpUsd": 5500, "sourceNote": "Ampere workstation.", "sourceUrl": "https://www.nvidia.com/en-us/data-center/a40/", "testedAt": "2024-10-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000908, "perfPerWatt": 0.213, "tokensPerDollar": 0.012, "freshnessDays": 637, "isStale": true }, { "id": "bench-a5000-l3-8b-q4", "deviceId": "rtx-a5000", "deviceName": "NVIDIA RTX A5000 24GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 230, "msrpUsd": 2200, "sourceNote": "Ampere pro.", "sourceUrl": "https://www.nvidia.com/en-us/design-visualization/rtx-a5000/", "testedAt": "2024-08-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000401, "perfPerWatt": 0.252, "tokensPerDollar": 0.026, "freshnessDays": 680, "isStale": true }, { "id": "bench-a4000-l3-8b-q4", "deviceId": "rtx-a4000", "deviceName": "NVIDIA RTX A4000 16GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 140, "msrpUsd": 1000, "sourceNote": "Single-slot pro card.", "sourceUrl": "https://www.nvidia.com/en-us/design-visualization/rtx-a4000/", "testedAt": "2024-08-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000252, "perfPerWatt": 0.3, "tokensPerDollar": 0.042, "freshnessDays": 691, "isStale": true }, { "id": "bench-a100-80gb-l3-70b-q4", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 80GB SXM", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "Reference vLLM 0.6.x.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2024-08-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003303, "perfPerWatt": 0.12, "tokensPerDollar": 0.003, "freshnessDays": 681, "isStale": true }, { "id": "bench-a100-80gb-l3-8b-fp16", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 80GB SXM", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 215, "quantization": "FP16", "contextLength": 4096, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "vLLM reference.", "sourceUrl": "https://mlcommons.org/benchmarks/inference-datacenter/", "testedAt": "2024-09-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000737, "perfPerWatt": 0.538, "tokensPerDollar": 0.014, "freshnessDays": 664, "isStale": true }, { "id": "bench-a100-40gb-l3-70b-q4", "deviceId": "a100-40gb", "deviceName": "NVIDIA A100 40GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 40, "tdpW": 400, "msrpUsd": 9000, "sourceNote": "MLPerf inference reference.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2024-09-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002265, "perfPerWatt": 0.105, "tokensPerDollar": 0.005, "freshnessDays": 676, "isStale": true }, { "id": "bench-a100-40gb-l3-8b-fp16", "deviceId": "a100-40gb", "deviceName": "NVIDIA A100 40GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 195, "quantization": "FP16", "contextLength": 4096, "vramGB": 40, "tdpW": 400, "msrpUsd": 9000, "sourceNote": "MLPerf.", "sourceUrl": "https://mlcommons.org/benchmarks/inference-datacenter/", "testedAt": "2024-09-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000488, "perfPerWatt": 0.488, "tokensPerDollar": 0.022, "freshnessDays": 669, "isStale": true }, { "id": "bench-gh200-l3-70b-q4", "deviceId": "gh200", "deviceName": "NVIDIA GH200 Grace Hopper 480GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 95, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 1000, "msrpUsd": 44000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-02-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.004896, "perfPerWatt": 0.095, "tokensPerDollar": 0.002, "freshnessDays": 507, "isStale": true }, { "id": "bench-gh200-l3-8b-fp16", "deviceId": "gh200", "deviceName": "NVIDIA GH200 Grace Hopper 480GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 380, "quantization": "FP16", "contextLength": 8192, "vramGB": 96, "tdpW": 1000, "msrpUsd": 44000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-02-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001224, "perfPerWatt": 0.38, "tokensPerDollar": 0.009, "freshnessDays": 500, "isStale": true }, { "id": "bench-mi250x-l3-70b-q4", "deviceId": "mi250x", "deviceName": "AMD Instinct MI250X 128GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 560, "msrpUsd": 14000, "sourceNote": "ROCm 6.0 + vLLM.", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2024-11-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002846, "perfPerWatt": 0.093, "tokensPerDollar": 0.004, "freshnessDays": 605, "isStale": true }, { "id": "bench-mi250x-l3-8b-fp16", "deviceId": "mi250x", "deviceName": "AMD Instinct MI250X 128GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 188, "quantization": "FP16", "contextLength": 4096, "vramGB": 128, "tdpW": 560, "msrpUsd": 14000, "sourceNote": "ROCm 6.0.", "sourceUrl": "https://www.amd.com/en/developer/resources/ml-models.html", "testedAt": "2024-11-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000787, "perfPerWatt": 0.336, "tokensPerDollar": 0.013, "freshnessDays": 598, "isStale": true }, { "id": "bench-mi210-l3-70b-q4", "deviceId": "mi210", "deviceName": "AMD Instinct MI210 64GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 300, "msrpUsd": 8000, "sourceNote": "ROCm 5.7.", "sourceUrl": "https://www.amd.com/en/developer/resources/ml-models.html", "testedAt": "2024-09-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00302, "perfPerWatt": 0.093, "tokensPerDollar": 0.004, "freshnessDays": 648, "isStale": true }, { "id": "bench-rx7700xt-l3-8b-q4", "deviceId": "rx-7700-xt", "deviceName": "AMD Radeon RX 7700 XT 12GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 245, "msrpUsd": 449, "sourceNote": "llama.cpp Vulkan.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1aym10w/7900_xtx_rocm_results/", "testedAt": "2024-11-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000091, "perfPerWatt": 0.212, "tokensPerDollar": 0.116, "freshnessDays": 613, "isStale": true }, { "id": "bench-rx7700xt-mistral", "deviceId": "rx-7700-xt", "deviceName": "AMD Radeon RX 7700 XT 12GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 245, "msrpUsd": 449, "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5853", "testedAt": "2024-11-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000082, "perfPerWatt": 0.237, "tokensPerDollar": 0.129, "freshnessDays": 596, "isStale": true }, { "id": "bench-rx7600-l3-8b-q4", "deviceId": "rx-7600", "deviceName": "AMD Radeon RX 7600 8GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 165, "msrpUsd": 269, "sourceNote": "Tight VRAM.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1aym10w/7900_xtx_rocm_results/", "testedAt": "2024-09-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000102, "perfPerWatt": 0.17, "tokensPerDollar": 0.104, "freshnessDays": 656, "isStale": true }, { "id": "bench-rx7600-phi3", "deviceId": "rx-7600", "deviceName": "AMD Radeon RX 7600 8GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 165, "msrpUsd": 269, "sourceNote": "Phi-3 fits comfortably.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-10-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000049, "perfPerWatt": 0.352, "tokensPerDollar": 0.216, "freshnessDays": 636, "isStale": true }, { "id": "bench-arc-a770-l3-8b-q4", "deviceId": "arc-a770", "deviceName": "Intel Arc A770 16GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 36, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 225, "msrpUsd": 329, "sourceNote": "IPEX-LLM Q4, single batch.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2024-10-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000097, "perfPerWatt": 0.16, "tokensPerDollar": 0.109, "freshnessDays": 623, "isStale": true }, { "id": "bench-arc-a770-mistral", "deviceId": "arc-a770", "deviceName": "Intel Arc A770 16GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 40, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 225, "msrpUsd": 329, "sourceNote": "IPEX-LLM.", "sourceUrl": "https://www.intel.com/content/www/us/en/developer/articles/technical/intel-arc-llm-benchmarks.html", "testedAt": "2024-11-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000087, "perfPerWatt": 0.178, "tokensPerDollar": 0.122, "freshnessDays": 615, "isStale": true }, { "id": "bench-arc-a770-sdxl", "deviceId": "arc-a770", "deviceName": "Intel Arc A770 16GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 5, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 225, "msrpUsd": 329, "sourceNote": "OpenVINO.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2024-11-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 602, "isStale": true }, { "id": "bench-arc-a750-l3-8b-q4", "deviceId": "arc-a750", "deviceName": "Intel Arc A750 8GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 225, "msrpUsd": 249, "sourceNote": "IPEX-LLM; 8GB tight for 8B at full 4K context.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2024-10-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00011, "perfPerWatt": 0.107, "tokensPerDollar": 0.096, "freshnessDays": 640, "isStale": true }, { "id": "bench-m3-max-l3-8b-q4", "deviceId": "apple-m3-max", "deviceName": "Apple M3 Max (40c GPU, 64GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 50, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 64, "tdpW": 65, "msrpUsd": 3499, "sourceNote": "llama.cpp Metal, MLX comparable.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1cfp1ej/m4_max_llama_benchmarks/", "testedAt": "2024-12-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00074, "perfPerWatt": 0.769, "tokensPerDollar": 0.014, "freshnessDays": 585, "isStale": true }, { "id": "bench-m3-max-l3-70b-q4", "deviceId": "apple-m3-max", "deviceName": "Apple M3 Max (40c GPU, 64GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q4", "tokensPerSec": 5.5, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 65, "msrpUsd": 3499, "sourceNote": "Unified memory, Metal.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-12-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.006724, "perfPerWatt": 0.085, "tokensPerDollar": 0.002, "freshnessDays": 572, "isStale": true }, { "id": "bench-m3-pro-l3-8b-q4", "deviceId": "apple-m3-pro", "deviceName": "Apple M3 Pro (18c GPU, 36GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 36, "tdpW": 50, "msrpUsd": 2499, "sourceNote": "M3 Pro 12-core CPU + 18c GPU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-11-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001201, "perfPerWatt": 0.44, "tokensPerDollar": 0.009, "freshnessDays": 609, "isStale": true }, { "id": "bench-m3-pro-mistral", "deviceId": "apple-m3-pro", "deviceName": "Apple M3 Pro (18c GPU, 36GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 25, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 36, "tdpW": 50, "msrpUsd": 2499, "sourceNote": "llama.cpp Metal.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2024-11-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001057, "perfPerWatt": 0.5, "tokensPerDollar": 0.01, "freshnessDays": 605, "isStale": true }, { "id": "bench-m2-pro-l3-8b-q4", "deviceId": "apple-m2-pro", "deviceName": "Apple M2 Pro (19c GPU, 32GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 20, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 45, "msrpUsd": 1999, "sourceNote": "Community-submitted llama.cpp Metal.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1cj5um6/m2_pro_llama3_metal_benchmarks/", "testedAt": "2024-08-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001056, "perfPerWatt": 0.444, "tokensPerDollar": 0.01, "freshnessDays": 695, "isStale": true }, { "id": "bench-m4-l3-8b-q4", "deviceId": "apple-m4", "deviceName": "Apple M4 (10c GPU, 24GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 22, "msrpUsd": 1599, "sourceNote": "iPad/MacBook Air class M4 base.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-12-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000939, "perfPerWatt": 0.818, "tokensPerDollar": 0.011, "freshnessDays": 566, "isStale": true }, { "id": "bench-m4-pro-l3-8b-q4", "deviceId": "apple-m4-pro", "deviceName": "Apple M4 Pro (20c GPU, 48GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 50, "msrpUsd": 2499, "sourceNote": "M4 Pro MacBook Pro 14\".", "sourceUrl": "https://github.com/ml-explore/mlx-examples", "testedAt": "2025-01-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000695, "perfPerWatt": 0.76, "tokensPerDollar": 0.015, "freshnessDays": 552, "isStale": true }, { "id": "bench-m4-pro-mistral", "deviceId": "apple-m4-pro", "deviceName": "Apple M4 Pro (20c GPU, 48GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 50, "msrpUsd": 2499, "sourceNote": "MLX 0.18.", "sourceUrl": "https://github.com/ml-explore/mlx-examples", "testedAt": "2025-01-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000629, "perfPerWatt": 0.84, "tokensPerDollar": 0.017, "freshnessDays": 545, "isStale": true }, { "id": "bench-sdx-elite-whisper", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite (X1E-84-100)", "brand": "Qualcomm", "deviceClass": "npu", "workload": "whisper-large-v3", "audioRtfx": 2.1, "quantization": "INT8", "contextLength": 0, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "amazonAsin": "B0CXL8T7GC", "sourceNote": "QNN SDK, Hexagon NPU.", "sourceUrl": "https://github.com/quic/ai-hub-models", "testedAt": "2024-10-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 634, "isStale": true }, { "id": "bench-ryzenai-365-phi3", "deviceId": "amd-ryzen-ai-9-365", "deviceName": "AMD Ryzen AI 9 365 NPU", "brand": "AMD", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 16, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 28, "msrpUsd": 1199, "sourceNote": "Ryzen AI SW 1.3, 50 TOPS XDNA 2.", "sourceUrl": "https://www.amd.com/en/products/processors/laptop/ryzen/ai-300-series.html", "testedAt": "2024-12-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000792, "perfPerWatt": 0.571, "tokensPerDollar": 0.013, "freshnessDays": 578, "isStale": true }, { "id": "bench-ryzenai-365-l3-8b-q4", "deviceId": "amd-ryzen-ai-9-365", "deviceName": "AMD Ryzen AI 9 365 NPU", "brand": "AMD", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 6, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 28, "msrpUsd": 1199, "sourceNote": "Hybrid NPU + iGPU.", "sourceUrl": "https://github.com/amd/RyzenAI-SW", "testedAt": "2024-12-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002112, "perfPerWatt": 0.214, "tokensPerDollar": 0.005, "freshnessDays": 569, "isStale": true }, { "id": "bench-orin-nano-super-phi3", "deviceId": "jetson-orin-nano-super", "deviceName": "NVIDIA Jetson Orin Nano Super 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 25, "msrpUsd": 249, "sourceNote": "Orin Nano \"Super\" Dec-2024 firmware.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000082, "perfPerWatt": 1.28, "tokensPerDollar": 0.129, "freshnessDays": 548, "isStale": true }, { "id": "bench-orin-nano-super-l3-8b-q4", "deviceId": "jetson-orin-nano-super", "deviceName": "NVIDIA Jetson Orin Nano Super 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "llama3-8b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 25, "msrpUsd": 249, "sourceNote": "Orin Nano Super refresh.", "sourceUrl": "https://developer.nvidia.com/embedded/jetson-benchmarks", "testedAt": "2025-01-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000188, "perfPerWatt": 0.56, "tokensPerDollar": 0.056, "freshnessDays": 541, "isStale": true }, { "id": "bench-agx-orin-l3-70b-q4", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "llama3-70b-q4", "tokensPerSec": 4.2, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "TensorRT-LLM, MAXN.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2024-10-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.005031, "perfPerWatt": 0.07, "tokensPerDollar": 0.002, "freshnessDays": 619, "isStale": true }, { "id": "bench-agx-orin-mistral", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "mistral-7b-q4", "tokensPerSec": 25, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "TensorRT-LLM.", "sourceUrl": "https://developer.nvidia.com/embedded/jetson-benchmarks", "testedAt": "2024-11-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000845, "perfPerWatt": 0.417, "tokensPerDollar": 0.013, "freshnessDays": 613, "isStale": true }, { "id": "bench-pi5-aihat-l3-8b-q4", "deviceId": "raspberry-pi-ai-hat-plus", "deviceName": "Raspberry Pi 5 + AI HAT+ (Hailo-8)", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "llama3-8b-q4", "tokensPerSec": 1.2, "quantization": "INT8", "contextLength": 1024, "vramGB": 8, "tdpW": 12, "msrpUsd": 150, "amazonAsin": "B0D8Q6GNM2", "sourceNote": "Hailo-8 26 TOPS — partial CPU fallback.", "sourceUrl": "https://github.com/hailo-ai/hailo-rpi5-examples", "testedAt": "2024-09-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001321, "perfPerWatt": 0.1, "tokensPerDollar": 0.008, "freshnessDays": 652, "isStale": true }, { "id": "bench-groq-mixtral", "deviceId": "groq-lpu", "deviceName": "Groq LPU (8-chip rack)", "brand": "Groq", "deviceClass": "asic", "workload": "qwen2.5-14b-q4", "tokensPerSec": 480, "quantization": "FP16", "contextLength": 8192, "vramGB": 1.84, "tdpW": 1720, "msrpUsd": 160000, "sourceNote": "Vendor-published GroqCloud number.", "sourceUrl": "https://wow.groq.com/", "testedAt": "2024-08-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003523, "perfPerWatt": 0.279, "tokensPerDollar": 0.003, "freshnessDays": 687, "isStale": true }, { "id": "bench-groq-deepseek", "deviceId": "groq-lpu", "deviceName": "Groq LPU (per chip)", "brand": "Groq", "deviceClass": "asic", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 620, "quantization": "FP16", "contextLength": 8192, "vramGB": 0.23, "tdpW": 215, "msrpUsd": 20000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://wow.groq.com/", "testedAt": "2025-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000341, "perfPerWatt": 2.884, "tokensPerDollar": 0.031, "freshnessDays": 521, "isStale": true }, { "id": "bench-cerebras-mistral", "deviceId": "cerebras-wse-3", "deviceName": "Cerebras WSE-3 (CS-3)", "brand": "Cerebras", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 1620, "quantization": "FP16", "contextLength": 8192, "vramGB": 44, "tdpW": 23000, "msrpUsd": 2500000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://cerebras.ai/blog/cerebras-inference-llama-3-1-70b", "testedAt": "2024-09-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.016312, "perfPerWatt": 0.07, "tokensPerDollar": 0.001, "freshnessDays": 666, "isStale": true }, { "id": "bench-tpu-v5e-l3-8b-fp16", "deviceId": "tpu-v5e", "deviceName": "Google TPU v5e", "brand": "Google", "deviceClass": "asic", "workload": "llama3-8b-fp16", "tokensPerSec": 240, "quantization": "FP16", "contextLength": 4096, "vramGB": 16, "tdpW": 170, "msrpUsd": 9000, "sourceNote": "Vendor benchmark — JAX/MaxText.", "sourceUrl": "https://cloud.google.com/blog/products/compute/introducing-trillium-6th-gen-tpus", "testedAt": "2024-08-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000396, "perfPerWatt": 1.412, "tokensPerDollar": 0.027, "freshnessDays": 701, "isStale": true }, { "id": "bench-tpu-v5e-l3-70b-q8", "deviceId": "tpu-v5e", "deviceName": "Google TPU v5e", "brand": "Google", "deviceClass": "asic", "workload": "llama3-70b-q8", "tokensPerSec": 62, "quantization": "INT8", "contextLength": 4096, "vramGB": 16, "tdpW": 170, "msrpUsd": 9000, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://github.com/google/maxtext", "testedAt": "2024-08-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001534, "perfPerWatt": 0.365, "tokensPerDollar": 0.007, "freshnessDays": 694, "isStale": true }, { "id": "bench-2x4090-l3-70b-q4", "deviceId": "rtx-4090-2x", "deviceName": "2× NVIDIA RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 900, "msrpUsd": 3200, "sourceNote": "Tensor-parallel via vLLM. Power = 2x card TDP.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-03-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001208, "perfPerWatt": 0.031, "tokensPerDollar": 0.009, "freshnessDays": 482, "isStale": true }, { "id": "bench-2x4090-l3-70b-q8", "deviceId": "rtx-4090-2x", "deviceName": "2× NVIDIA RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 18, "quantization": "Q8_0", "contextLength": 4096, "vramGB": 48, "tdpW": 900, "msrpUsd": 3200, "sourceNote": "Tensor-parallel.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-03-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001879, "perfPerWatt": 0.02, "tokensPerDollar": 0.006, "freshnessDays": 475, "isStale": true }, { "id": "bench-4x3090-l3-70b-q4", "deviceId": "rtx-3090-4x", "deviceName": "4× NVIDIA RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 1400, "msrpUsd": 4000, "sourceNote": "Used-3090 quad popular on r/LocalLLaMA.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d9qjcm/mi300x_benchmarks/", "testedAt": "2024-12-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001762, "perfPerWatt": 0.017, "tokensPerDollar": 0.006, "freshnessDays": 557, "isStale": true }, { "id": "bench-4x3090-l3-70b-q8", "deviceId": "rtx-3090-4x", "deviceName": "4× NVIDIA RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 14, "quantization": "Q8_0", "contextLength": 4096, "vramGB": 96, "tdpW": 1400, "msrpUsd": 4000, "sourceNote": "Tensor-parallel via vLLM.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1bz9pek/4x3090_70b_q8_results/", "testedAt": "2025-01-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00302, "perfPerWatt": 0.01, "tokensPerDollar": 0.004, "freshnessDays": 537, "isStale": true }, { "id": "bench-8xh100-l3-70b-q4", "deviceId": "h100-sxm5-8x", "deviceName": "8× NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 412, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 640, "tdpW": 5600, "msrpUsd": 200000, "sourceNote": "DGX H100 reference, vLLM TP=8.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2024-09-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.005131, "perfPerWatt": 0.074, "tokensPerDollar": 0.002, "freshnessDays": 658, "isStale": true }, { "id": "bench-8xh100-l3-8b-fp16", "deviceId": "h100-sxm5-8x", "deviceName": "8× NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 1980, "quantization": "FP16", "contextLength": 8192, "vramGB": 640, "tdpW": 5600, "msrpUsd": 200000, "sourceNote": "DGX H100, vLLM TP=8.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2024-10-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001068, "perfPerWatt": 0.354, "tokensPerDollar": 0.01, "freshnessDays": 644, "isStale": true }, { "id": "bench-rtx3090-qwen-14b", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 55, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000288, "perfPerWatt": 0.157, "tokensPerDollar": 0.037, "freshnessDays": 489, "isStale": true }, { "id": "bench-rtx3090-gemma", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 65, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Community-submitted.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2024-08-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000244, "perfPerWatt": 0.186, "tokensPerDollar": 0.043, "freshnessDays": 679, "isStale": true }, { "id": "bench-rtx3090-deepseek", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 82, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Community-submitted.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-01-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000193, "perfPerWatt": 0.234, "tokensPerDollar": 0.055, "freshnessDays": 527, "isStale": true }, { "id": "bench-rtx4080s-qwen-14b", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 62, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Community-submitted.", "sourceUrl": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF", "testedAt": "2024-12-13", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00017, "perfPerWatt": 0.194, "tokensPerDollar": 0.062, "freshnessDays": 574, "isStale": true }, { "id": "bench-rtx4080s-deepseek", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 98, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Community-submitted.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1ihy3mc/deepseek_r1_distill_7b_on_4080_super/", "testedAt": "2025-02-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000108, "perfPerWatt": 0.306, "tokensPerDollar": 0.098, "freshnessDays": 510, "isStale": true }, { "id": "bench-rx7900xtx-qwen-14b", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "amazonAsin": "B0BPRH5RNN", "sourceNote": "Community-submitted, ROCm.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1aym10w/7900_xtx_rocm_results/", "testedAt": "2024-11-27", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00022, "perfPerWatt": 0.135, "tokensPerDollar": 0.048, "freshnessDays": 590, "isStale": true }, { "id": "bench-mi300x-qwen-14b", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 142, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2.", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2024-12-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00134, "perfPerWatt": 0.189, "tokensPerDollar": 0.008, "freshnessDays": 583, "isStale": true }, { "id": "bench-h100-qwen-14b", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 168, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "vLLM 0.6.x.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2024-10-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001573, "perfPerWatt": 0.24, "tokensPerDollar": 0.007, "freshnessDays": 627, "isStale": true }, { "id": "bench-rtx4070-whisper", "deviceId": "rtx-4070", "deviceName": "NVIDIA GeForce RTX 4070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 42, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 200, "msrpUsd": 599, "sourceNote": "faster-whisper.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/blob/master/README.md#benchmark", "testedAt": "2024-09-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 674, "isStale": true }, { "id": "bench-rtx4060ti-16gb-whisper", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 30, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "faster-whisper.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/blob/master/README.md#benchmark", "testedAt": "2024-09-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 669, "isStale": true }, { "id": "bench-rx7900xtx-whisper", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 48, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "amazonAsin": "B0BPRH5RNN", "sourceNote": "whisper.cpp ROCm.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2024-09-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 648, "isStale": true }, { "id": "bench-m3-max-whisper", "deviceId": "apple-m3-max", "deviceName": "Apple M3 Max (40c GPU, 64GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "whisper-large-v3", "audioRtfx": 26, "quantization": "FP16", "contextLength": 0, "vramGB": 64, "tdpW": 65, "msrpUsd": 3499, "sourceNote": "whisper.cpp Metal.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2024-10-17", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 631, "isStale": true }, { "id": "bench-orin-nano-super-whisper", "deviceId": "jetson-orin-nano-super", "deviceName": "NVIDIA Jetson Orin Nano Super 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "whisper-large-v3", "audioRtfx": 4.5, "quantization": "FP16", "contextLength": 0, "vramGB": 8, "tdpW": 25, "msrpUsd": 249, "sourceNote": "faster-whisper INT8.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-01-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 534, "isStale": true }, { "id": "bench-l40s-embed", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 7400, "quantization": "FP16", "contextLength": 512, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Text-embeddings-inference batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-08-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 683, "isStale": true }, { "id": "bench-a100-80gb-embed", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 80GB SXM", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 9100, "quantization": "FP16", "contextLength": 512, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-09-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 653, "isStale": true }, { "id": "bench-rtx3090-embed", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 3200, "quantization": "FP16", "contextLength": 512, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-10-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 639, "isStale": true }, { "id": "bench-m3ultra-l3-70b-q8", "deviceId": "apple-m2-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q8", "tokensPerSec": 9, "quantization": "Q8_0", "contextLength": 8192, "vramGB": 512, "tdpW": 270, "msrpUsd": 9499, "sourceNote": "Unified memory; CPU+GPU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-04-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.011156, "perfPerWatt": 0.033, "tokensPerDollar": 0.001, "freshnessDays": 464, "isStale": true }, { "id": "bench-m3ultra-qwen-14b", "deviceId": "apple-m2-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 512, "tdpW": 270, "msrpUsd": 9499, "sourceNote": "MLX.", "sourceUrl": "https://github.com/ml-explore/mlx-examples", "testedAt": "2025-04-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002642, "perfPerWatt": 0.141, "tokensPerDollar": 0.004, "freshnessDays": 456, "isStale": true }, { "id": "bench-rtx5090-mistral-v2", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 198, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "llama.cpp b3850, post-Blackwell driver tuning.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2026-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000107, "perfPerWatt": 0.344, "tokensPerDollar": 0.099, "freshnessDays": 148, "isStale": false }, { "id": "bench-rtx5090-phi3", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 320, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Small model fits with massive room to spare.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2026-03-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000066, "perfPerWatt": 0.557, "tokensPerDollar": 0.16, "freshnessDays": 131, "isStale": false }, { "id": "bench-rtx5090-l3-70b-q8", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 14, "quantization": "Q8_0", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Heavy CPU offload.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/6017", "testedAt": "2026-04-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001509, "perfPerWatt": 0.024, "tokensPerDollar": 0.007, "freshnessDays": 93, "isStale": false }, { "id": "bench-b200-qwen-14b", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 285, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2026-01-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001483, "perfPerWatt": 0.285, "tokensPerDollar": 0.007, "freshnessDays": 176, "isStale": false }, { "id": "bench-b200-deepseek", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 480, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2026-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000881, "perfPerWatt": 0.48, "tokensPerDollar": 0.012, "freshnessDays": 156, "isStale": false }, { "id": "bench-b200-whisper", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "whisper-large-v3", "audioRtfx": 210, "quantization": "FP16", "contextLength": 0, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Vendor benchmark.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2026-02-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 138, "isStale": false }, { "id": "bench-mi300x-deepseek", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 215, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2.", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2026-03-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000885, "perfPerWatt": 0.287, "tokensPerDollar": 0.012, "freshnessDays": 113, "isStale": false }, { "id": "bench-h200-mistral", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 305, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "TensorRT-LLM, HBM3e bandwidth.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2026-01-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00104, "perfPerWatt": 0.436, "tokensPerDollar": 0.01, "freshnessDays": 163, "isStale": false }, { "id": "bench-h200-sdxl", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "sdxl-1024", "imagesPerMin": 36, "quantization": "FP16", "contextLength": 0, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Diffusers + TensorRT.", "sourceUrl": "https://github.com/NVIDIA/TensorRT/tree/release/9.2/demo/Diffusion", "testedAt": "2026-02-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 145, "isStale": false }, { "id": "bench-h200-deepseek", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 305, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "TRT-LLM.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2026-04-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00104, "perfPerWatt": 0.436, "tokensPerDollar": 0.01, "freshnessDays": 81, "isStale": false }, { "id": "bench-mi300x-gemma", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 188, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2.", "sourceUrl": "https://github.com/ROCm/rocm-blogs/blob/release/blogs/artificial-intelligence/llama3/README.md", "testedAt": "2025-12-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001012, "perfPerWatt": 0.251, "tokensPerDollar": 0.01, "freshnessDays": 214, "isStale": true }, { "id": "bench-mi300x-mistral", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 268, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 192, "tdpW": 750, "msrpUsd": 18000, "sourceNote": "ROCm 6.2.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/5138", "testedAt": "2025-05-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00071, "perfPerWatt": 0.357, "tokensPerDollar": 0.015, "freshnessDays": 417, "isStale": true }, { "id": "bench-rtx4090-qwen-14b", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 68, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Community-submitted.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-07-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000249, "perfPerWatt": 0.151, "tokensPerDollar": 0.043, "freshnessDays": 371, "isStale": true }, { "id": "bench-rtx5090-embed-v2", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 9100, "quantization": "FP16", "contextLength": 512, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "TEI batch 64 (tuned).", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2026-05-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 49, "isStale": false }, { "id": "bench-m4-pro-whisper", "deviceId": "apple-m4-pro", "deviceName": "Apple M4 Pro (20c GPU, 48GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "whisper-large-v3", "audioRtfx": 22, "quantization": "FP16", "contextLength": 0, "vramGB": 48, "tdpW": 50, "msrpUsd": 2499, "sourceNote": "whisper.cpp Metal.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2026-05-24", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 47, "isStale": false }, { "id": "bench-m4-pro-sdxl", "deviceId": "apple-m4-pro", "deviceName": "Apple M4 Pro (20c GPU, 48GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "sdxl-1024", "imagesPerMin": 3.2, "quantization": "FP16", "contextLength": 0, "vramGB": 48, "tdpW": 50, "msrpUsd": 2499, "sourceNote": "DrawThings.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-05-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 46, "isStale": false }, { "id": "bench-rtx4090-deepseek-v2", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 125, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "llama.cpp b3850, refreshed seed.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2026-05-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000135, "perfPerWatt": 0.278, "tokensPerDollar": 0.078, "freshnessDays": 46, "isStale": false }, { "id": "bench-rtx5090-qwen-v2", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 105, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Driver R570 + GDDR7 tuning.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9hd6e/rtx_5090_inference_results/", "testedAt": "2026-05-24", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000201, "perfPerWatt": 0.183, "tokensPerDollar": 0.053, "freshnessDays": 47, "isStale": false }, { "id": "bench-rtx5080-l3-8b-q4", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 165, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "GDDR7 960 GB/s, R570 driver.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5080/", "testedAt": "2026-02-08", "batchSize": 1, "runsCompleted": 5, "stdDev": 4.8, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000064, "perfPerWatt": 0.458, "tokensPerDollar": 0.165, "freshnessDays": 152, "isStale": false, "minObserved": 114, "maxObserved": 128, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8", "osVersion": "Ubuntu 24.04" }, { "id": "bench-rtx5080-l3-8b-fp16", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 88, "quantization": "FP16", "contextLength": 4096, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "vLLM 0.6.4.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2026-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00012, "perfPerWatt": 0.244, "tokensPerDollar": 0.088, "freshnessDays": 148, "isStale": false }, { "id": "bench-rtx5080-l3-70b-q4", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 7, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Heavy CPU offload — does not fit in 16GB.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/6017", "testedAt": "2026-02-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001508, "perfPerWatt": 0.019, "tokensPerDollar": 0.007, "freshnessDays": 145, "isStale": false }, { "id": "bench-rtx5080-deepseek", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Distilled model fits comfortably.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-02-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000077, "perfPerWatt": 0.383, "tokensPerDollar": 0.138, "freshnessDays": 140, "isStale": false }, { "id": "bench-rtx5080-qwen-14b", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 82, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Tight VRAM fit at 8K ctx.", "sourceUrl": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF", "testedAt": "2026-02-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000129, "perfPerWatt": 0.228, "tokensPerDollar": 0.082, "freshnessDays": 138, "isStale": false }, { "id": "bench-rtx5080-sdxl", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 18, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "ComfyUI + TensorRT.", "sourceUrl": "https://developer.nvidia.com/blog/getting-started-with-stable-diffusion-tensorrt/", "testedAt": "2026-02-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 135, "isStale": false }, { "id": "bench-rtx5080-whisper", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 95, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "faster-whisper CTranslate2 INT8.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2026-03-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 131, "isStale": false }, { "id": "bench-rtx5080-embed", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 6800, "quantization": "FP16", "contextLength": 512, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2026-03-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 127, "isStale": false }, { "id": "bench-rtx5070ti-l3-8b-q4", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 128, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "GDDR7 896 GB/s effective.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5070-family/", "testedAt": "2026-02-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000062, "perfPerWatt": 0.427, "tokensPerDollar": 0.171, "freshnessDays": 140, "isStale": false }, { "id": "bench-rtx5070ti-deepseek", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 108, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "Distilled model.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-02-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000073, "perfPerWatt": 0.36, "tokensPerDollar": 0.144, "freshnessDays": 135, "isStale": false }, { "id": "bench-rtx5070ti-qwen-14b", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 65, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "Tight at 8K, OK at 4K.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2026-03-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000122, "perfPerWatt": 0.217, "tokensPerDollar": 0.087, "freshnessDays": 131, "isStale": false }, { "id": "bench-rtx5070ti-sdxl", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 14, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "ComfyUI 30 steps.", "sourceUrl": "https://developer.nvidia.com/blog/getting-started-with-stable-diffusion-tensorrt/", "testedAt": "2026-03-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 128, "isStale": false }, { "id": "bench-rtx5070-l3-8b-q4", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "12GB GDDR7, mid-range Blackwell.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5070-family/", "testedAt": "2026-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000063, "perfPerWatt": 0.368, "tokensPerDollar": 0.168, "freshnessDays": 124, "isStale": false }, { "id": "bench-rtx5070-deepseek", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 75, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Fits at 4K ctx.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-03-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000077, "perfPerWatt": 0.3, "tokensPerDollar": 0.137, "freshnessDays": 120, "isStale": false }, { "id": "bench-rtx5070-mistral", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 85, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Sweet spot for 7B models.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2026-03-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000068, "perfPerWatt": 0.34, "tokensPerDollar": 0.155, "freshnessDays": 117, "isStale": false }, { "id": "bench-rtx5070-phi3", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 165, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Small model flies on 12GB.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2026-03-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000035, "perfPerWatt": 0.66, "tokensPerDollar": 0.301, "freshnessDays": 116, "isStale": false }, { "id": "bench-rtx5070-gemma", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Gemma 2 9B Q4 fits.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2026-03-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000081, "perfPerWatt": 0.288, "tokensPerDollar": 0.131, "freshnessDays": 114, "isStale": false }, { "id": "bench-rtx5070-sdxl", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 10.5, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Tight VRAM at SDXL 1024.", "sourceUrl": "https://developer.nvidia.com/blog/getting-started-with-stable-diffusion-tensorrt/", "testedAt": "2026-03-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 112, "isStale": false }, { "id": "bench-rtx5060ti-16gb-l3-8b-q4", "deviceId": "rtx-5060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 180, "msrpUsd": 449, "sourceNote": "16GB GDDR7 variant — best entry value.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5060-family/", "testedAt": "2026-04-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000066, "perfPerWatt": 0.4, "tokensPerDollar": 0.16, "freshnessDays": 91, "isStale": false }, { "id": "bench-rtx5060ti-16gb-mistral", "deviceId": "rtx-5060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 68, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 180, "msrpUsd": 449, "sourceNote": "Great perf/W ratio.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2026-04-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00007, "perfPerWatt": 0.378, "tokensPerDollar": 0.151, "freshnessDays": 87, "isStale": false }, { "id": "bench-rtx5060ti-16gb-deepseek", "deviceId": "rtx-5060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 62, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 180, "msrpUsd": 449, "sourceNote": "Distilled R1 fits at 8K ctx.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-04-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000077, "perfPerWatt": 0.344, "tokensPerDollar": 0.138, "freshnessDays": 83, "isStale": false }, { "id": "bench-rtx5060ti-16gb-phi3", "deviceId": "rtx-5060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 132, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 180, "msrpUsd": 449, "sourceNote": "Best $/tok in the 50-series.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2026-04-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000036, "perfPerWatt": 0.733, "tokensPerDollar": 0.294, "freshnessDays": 79, "isStale": false }, { "id": "bench-rtx5060ti-8gb-l3-8b-q4", "deviceId": "rtx-5060-ti-8gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 165, "msrpUsd": 379, "sourceNote": "8GB tight — context length capped.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5060-family/", "testedAt": "2026-04-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000069, "perfPerWatt": 0.352, "tokensPerDollar": 0.153, "freshnessDays": 89, "isStale": false }, { "id": "bench-rtx5060ti-8gb-phi3", "deviceId": "rtx-5060-ti-8gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 118, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 165, "msrpUsd": 379, "sourceNote": "Phi-3 fits cleanly.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2026-04-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000034, "perfPerWatt": 0.715, "tokensPerDollar": 0.311, "freshnessDays": 85, "isStale": false }, { "id": "bench-rtx5060-l3-8b-q4", "deviceId": "rtx-5060", "deviceName": "NVIDIA GeForce RTX 5060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 145, "msrpUsd": 299, "sourceNote": "Entry-level Blackwell.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/50-series/rtx-5060-family/", "testedAt": "2026-04-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000066, "perfPerWatt": 0.331, "tokensPerDollar": 0.161, "freshnessDays": 76, "isStale": false }, { "id": "bench-rtx5060-phi3", "deviceId": "rtx-5060", "deviceName": "NVIDIA GeForce RTX 5060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 98, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 145, "msrpUsd": 299, "sourceNote": "Phi-3 fits.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2026-04-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000032, "perfPerWatt": 0.676, "tokensPerDollar": 0.328, "freshnessDays": 73, "isStale": false }, { "id": "bench-rtx4060-l3-8b-q4", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Ada AD107.", "sourceUrl": "https://www.nvidia.com/en-us/geforce/graphics-cards/40-series/rtx-4060-family/", "testedAt": "2024-08-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000075, "perfPerWatt": 0.365, "tokensPerDollar": 0.14, "freshnessDays": 691, "isStale": true }, { "id": "bench-rtx4060-phi3", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 88, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Best phi-3 perf at sub-$300.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-09-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000036, "perfPerWatt": 0.765, "tokensPerDollar": 0.294, "freshnessDays": 674, "isStale": true }, { "id": "bench-rtx4060-mistral", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Tight at 4K context.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2024-09-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000066, "perfPerWatt": 0.417, "tokensPerDollar": 0.161, "freshnessDays": 666, "isStale": true }, { "id": "bench-mac-mini-m4-16gb-l3-8b-q4", "deviceId": "mac-mini-m4-16gb", "deviceName": "Apple Mac mini M4 16GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 16, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 18, "msrpUsd": 599, "sourceNote": "Mac mini M4 base, 16GB unified.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-11-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000396, "perfPerWatt": 0.889, "tokensPerDollar": 0.027, "freshnessDays": 609, "isStale": true }, { "id": "bench-mac-mini-m4-16gb-phi3", "deviceId": "mac-mini-m4-16gb", "deviceName": "Apple Mac mini M4 16GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 18, "msrpUsd": 599, "sourceNote": "MLX framework.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-11-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000151, "perfPerWatt": 2.333, "tokensPerDollar": 0.07, "freshnessDays": 602, "isStale": true }, { "id": "bench-mac-mini-m4-16gb-mistral", "deviceId": "mac-mini-m4-16gb", "deviceName": "Apple Mac mini M4 16GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 18, "msrpUsd": 599, "sourceNote": "llama.cpp Metal.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-11-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000352, "perfPerWatt": 1, "tokensPerDollar": 0.03, "freshnessDays": 597, "isStale": true }, { "id": "bench-mac-mini-m4-24gb-l3-8b-q4", "deviceId": "mac-mini-m4-24gb", "deviceName": "Apple Mac mini M4 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 17, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 22, "msrpUsd": 999, "sourceNote": "Same chip, more unified memory for larger context.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-11-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000621, "perfPerWatt": 0.773, "tokensPerDollar": 0.017, "freshnessDays": 595, "isStale": true }, { "id": "bench-mac-mini-m4-24gb-gemma", "deviceId": "mac-mini-m4-24gb", "deviceName": "Apple Mac mini M4 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 22, "msrpUsd": 999, "sourceNote": "Gemma 2 9B fits with room to spare.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2024-12-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000754, "perfPerWatt": 0.636, "tokensPerDollar": 0.014, "freshnessDays": 586, "isStale": true }, { "id": "bench-mac-mini-m4pro-24gb-l3-8b-q4", "deviceId": "mac-mini-m4-pro-24gb", "deviceName": "Apple Mac mini M4 Pro 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 35, "msrpUsd": 1399, "sourceNote": "M4 Pro 16-core GPU, 273 GB/s mem bandwidth.", "sourceUrl": "https://www.apple.com/shop/buy-mac/mac-mini", "testedAt": "2024-11-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000462, "perfPerWatt": 0.914, "tokensPerDollar": 0.023, "freshnessDays": 592, "isStale": true }, { "id": "bench-mac-mini-m4pro-24gb-mistral", "deviceId": "mac-mini-m4-pro-24gb", "deviceName": "Apple Mac mini M4 Pro 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 36, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 35, "msrpUsd": 1399, "sourceNote": "MLX Mistral.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-12-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000411, "perfPerWatt": 1.029, "tokensPerDollar": 0.026, "freshnessDays": 579, "isStale": true }, { "id": "bench-mac-mini-m4pro-48gb-l3-70b-q4", "deviceId": "mac-mini-m4-pro-48gb", "deviceName": "Apple Mac mini M4 Pro 48GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q4", "tokensPerSec": 6.5, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "70B Q4 fits in 48GB unified.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2024-12-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003251, "perfPerWatt": 0.186, "tokensPerDollar": 0.003, "freshnessDays": 577, "isStale": true }, { "id": "bench-mac-mini-m4pro-48gb-qwen-14b", "deviceId": "mac-mini-m4-pro-48gb", "deviceName": "Apple Mac mini M4 Pro 48GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "Comfortable headroom.", "sourceUrl": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF", "testedAt": "2024-12-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001174, "perfPerWatt": 0.514, "tokensPerDollar": 0.009, "freshnessDays": 573, "isStale": true }, { "id": "bench-mac-mini-m4pro-48gb-whisper", "deviceId": "mac-mini-m4-pro-48gb", "deviceName": "Apple Mac mini M4 Pro 48GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "whisper-large-v3", "audioRtfx": 18, "quantization": "FP16", "contextLength": 0, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "whisper.cpp Metal.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2024-12-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 569, "isStale": true }, { "id": "bench-strix-halo-l3-8b-q4", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "RDNA 3.5 iGPU + XDNA 2 NPU + 256 GB/s unified LPDDR5X.", "sourceUrl": "https://www.amd.com/en/processors/ryzen-ai-max.html", "testedAt": "2026-01-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00083, "perfPerWatt": 0.233, "tokensPerDollar": 0.013, "freshnessDays": 179, "isStale": false }, { "id": "bench-strix-halo-l3-70b-q4", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "llama3-70b-q4", "tokensPerSec": 5, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "70B Q4 fits in 96GB unified — first APU to do so.", "sourceUrl": "https://www.amd.com/en/processors/ryzen-ai-max.html", "testedAt": "2026-01-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.004649, "perfPerWatt": 0.042, "tokensPerDollar": 0.002, "freshnessDays": 173, "isStale": false }, { "id": "bench-strix-halo-qwen-14b", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "Strix Halo iGPU via ROCm.", "sourceUrl": "https://github.com/ROCm/ROCm", "testedAt": "2026-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00166, "perfPerWatt": 0.117, "tokensPerDollar": 0.006, "freshnessDays": 156, "isStale": false }, { "id": "bench-strix-halo-mistral", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "Best mobile inference APU on the market.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1iwzckm/strix_halo_inference_results/", "testedAt": "2026-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000726, "perfPerWatt": 0.267, "tokensPerDollar": 0.015, "freshnessDays": 148, "isStale": false }, { "id": "bench-strix-halo-phi3", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "XDNA 2 NPU path.", "sourceUrl": "https://github.com/amd/RyzenAI-SW", "testedAt": "2026-02-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000298, "perfPerWatt": 0.65, "tokensPerDollar": 0.035, "freshnessDays": 140, "isStale": false }, { "id": "bench-strix-halo-whisper", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "whisper-large-v3", "audioRtfx": 8, "quantization": "INT8", "contextLength": 0, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "Whisper via ONNX Runtime + DirectML.", "sourceUrl": "https://github.com/microsoft/onnxruntime", "testedAt": "2026-03-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 130, "isStale": false }, { "id": "bench-dgx-spark-l3-70b-q4", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "GB10 Grace + Blackwell, 128GB unified LPDDR5X.", "sourceUrl": "https://www.nvidia.com/en-us/project-digits/", "testedAt": "2026-03-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000991, "perfPerWatt": 0.133, "tokensPerDollar": 0.011, "freshnessDays": 117, "isStale": false }, { "id": "bench-dgx-spark-l3-70b-q8", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 22, "quantization": "Q8_0", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "70B Q8 fits in 128GB unified.", "sourceUrl": "https://www.nvidia.com/en-us/project-digits/", "testedAt": "2026-03-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001441, "perfPerWatt": 0.092, "tokensPerDollar": 0.007, "freshnessDays": 112, "isStale": false }, { "id": "bench-dgx-spark-l3-8b-fp16", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-fp16", "tokensPerSec": 92, "quantization": "FP16", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "Workstation-class inference.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2026-03-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000345, "perfPerWatt": 0.383, "tokensPerDollar": 0.031, "freshnessDays": 107, "isStale": false }, { "id": "bench-dgx-spark-qwen-14b", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "Mid-size models on a desk.", "sourceUrl": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF", "testedAt": "2026-04-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000406, "perfPerWatt": 0.325, "tokensPerDollar": 0.026, "freshnessDays": 99, "isStale": false }, { "id": "bench-dgx-spark-deepseek", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 145, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "R1 distilled flies.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-04-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000219, "perfPerWatt": 0.604, "tokensPerDollar": 0.048, "freshnessDays": 96, "isStale": false }, { "id": "bench-arc-b580-deepseek", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "Battlemage IPEX-LLM, surprisingly competitive at this price.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2026-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000051, "perfPerWatt": 0.274, "tokensPerDollar": 0.209, "freshnessDays": 183, "isStale": true }, { "id": "bench-arc-b580-phi3", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 86, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "Phi-3 fits comfortably.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2026-01-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000031, "perfPerWatt": 0.453, "tokensPerDollar": 0.345, "freshnessDays": 176, "isStale": false }, { "id": "bench-arc-b580-gemma", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "Best Gemma value under $300.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2026-01-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000063, "perfPerWatt": 0.221, "tokensPerDollar": 0.169, "freshnessDays": 169, "isStale": false }, { "id": "bench-arc-b580-sdxl", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 6.2, "quantization": "FP16", "contextLength": 0, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "OpenVINO + Diffusers.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2026-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 156, "isStale": false }, { "id": "bench-arc-b580-whisper", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 28, "quantization": "INT8", "contextLength": 0, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "OpenVINO Whisper.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2026-02-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 145, "isStale": false }, { "id": "bench-m4-max-l3-70b-q4", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q4", "tokensPerSec": 8.5, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 65, "msrpUsd": 4999, "sourceNote": "MLX 0.20, 546 GB/s unified.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-01-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.006216, "perfPerWatt": 0.131, "tokensPerDollar": 0.002, "freshnessDays": 161, "isStale": false }, { "id": "bench-m4-max-l3-70b-q8", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-70b-q8", "tokensPerSec": 5.2, "quantization": "Q8_0", "contextLength": 4096, "vramGB": 128, "tdpW": 65, "msrpUsd": 4999, "sourceNote": "70B Q8 fits in 128GB unified.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2026-02-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.010161, "perfPerWatt": 0.08, "tokensPerDollar": 0.001, "freshnessDays": 152, "isStale": false }, { "id": "bench-m4-max-deepseek", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 128, "tdpW": 65, "msrpUsd": 4999, "sourceNote": "R1 distilled, room for huge ctx.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-02-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000911, "perfPerWatt": 0.892, "tokensPerDollar": 0.012, "freshnessDays": 138, "isStale": false }, { "id": "bench-m4-max-qwen-14b", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 128, "tdpW": 65, "msrpUsd": 4999, "sourceNote": "Massive context fits.", "sourceUrl": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF", "testedAt": "2026-03-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001391, "perfPerWatt": 0.585, "tokensPerDollar": 0.008, "freshnessDays": 128, "isStale": false }, { "id": "bench-m4-max-embed", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "embedding-bge-large", "embeddingsPerSec": 1850, "quantization": "FP16", "contextLength": 512, "vramGB": 128, "tdpW": 65, "msrpUsd": 4999, "sourceNote": "MLX TEI port, batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2026-03-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 120, "isStale": false }, { "id": "bench-tr-7995wx-l3-8b-q4", "deviceId": "amd-threadripper-7995wx", "deviceName": "AMD Threadripper PRO 7995WX (96-core)", "brand": "AMD", "deviceClass": "cpu-only", "workload": "llama3-8b-q4", "tokensPerSec": 12, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 350, "msrpUsd": 9999, "sourceNote": "Zen 4, 8-channel DDR5-5200, llama.cpp AVX-512.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4216", "testedAt": "2024-09-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.008807, "perfPerWatt": 0.034, "tokensPerDollar": 0.001, "freshnessDays": 656, "isStale": true }, { "id": "bench-tr-7995wx-l3-70b-q4", "deviceId": "amd-threadripper-7995wx", "deviceName": "AMD Threadripper PRO 7995WX (96-core)", "brand": "AMD", "deviceClass": "cpu-only", "workload": "llama3-70b-q4", "tokensPerSec": 2.4, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 350, "msrpUsd": 9999, "sourceNote": "70B Q4 fits in 256GB+ DDR5 — slow but functional.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1ai5x6q/threadripper_7995wx_70b_q4/", "testedAt": "2024-10-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.044037, "perfPerWatt": 0.007, "tokensPerDollar": 0, "freshnessDays": 644, "isStale": true }, { "id": "bench-tr-7995wx-phi3", "deviceId": "amd-threadripper-7995wx", "deviceName": "AMD Threadripper PRO 7995WX (96-core)", "brand": "AMD", "deviceClass": "cpu-only", "workload": "phi3-mini-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 350, "msrpUsd": 9999, "sourceNote": "CPU-only inference, 8-ch DDR5 bandwidth helps.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-10-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.003775, "perfPerWatt": 0.08, "tokensPerDollar": 0.003, "freshnessDays": 630, "isStale": true }, { "id": "bench-tr-7995wx-mistral", "deviceId": "amd-threadripper-7995wx", "deviceName": "AMD Threadripper PRO 7995WX (96-core)", "brand": "AMD", "deviceClass": "cpu-only", "workload": "mistral-7b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 350, "msrpUsd": 9999, "sourceNote": "Pure CPU, no GPU offload.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2024-11-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.007549, "perfPerWatt": 0.04, "tokensPerDollar": 0.001, "freshnessDays": 612, "isStale": true }, { "id": "bench-xeon-6-l3-8b-q4", "deviceId": "intel-xeon-6-6980p", "deviceName": "Intel Xeon 6980P (128c Granite Rapids)", "brand": "Intel", "deviceClass": "cpu-only", "workload": "llama3-8b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 500, "msrpUsd": 17800, "sourceNote": "AMX BF16 path, MRDIMM-8800.", "sourceUrl": "https://www.intel.com/content/www/us/en/products/sku/240777/intel-xeon-6980p-processor-504m-cache-2-00-ghz/specifications.html", "testedAt": "2025-04-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.013439, "perfPerWatt": 0.028, "tokensPerDollar": 0.001, "freshnessDays": 458, "isStale": true }, { "id": "bench-xeon-6-l3-70b-q4", "deviceId": "intel-xeon-6-6980p", "deviceName": "Intel Xeon 6980P (128c Granite Rapids)", "brand": "Intel", "deviceClass": "cpu-only", "workload": "llama3-70b-q4", "tokensPerSec": 2.8, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 500, "msrpUsd": 17800, "sourceNote": "12-channel DDR5 helps memory-bound inference.", "sourceUrl": "https://github.com/intel/intel-extension-for-transformers", "testedAt": "2025-04-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.067195, "perfPerWatt": 0.006, "tokensPerDollar": 0, "freshnessDays": 444, "isStale": true }, { "id": "bench-xeon-6-phi3", "deviceId": "intel-xeon-6-6980p", "deviceName": "Intel Xeon 6980P (128c Granite Rapids)", "brand": "Intel", "deviceClass": "cpu-only", "workload": "phi3-mini-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 500, "msrpUsd": 17800, "sourceNote": "AMX accelerated.", "sourceUrl": "https://github.com/intel/intel-extension-for-transformers", "testedAt": "2025-05-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00588, "perfPerWatt": 0.064, "tokensPerDollar": 0.002, "freshnessDays": 426, "isStale": true }, { "id": "bench-xeon-6-mistral", "deviceId": "intel-xeon-6-6980p", "deviceName": "Intel Xeon 6980P (128c Granite Rapids)", "brand": "Intel", "deviceClass": "cpu-only", "workload": "mistral-7b-q4", "tokensPerSec": 15, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 500, "msrpUsd": 17800, "sourceNote": "AMX BF16.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2025-05-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.012543, "perfPerWatt": 0.03, "tokensPerDollar": 0.001, "freshnessDays": 414, "isStale": true }, { "id": "bench-rtx4090-mistral", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Classic 7B benchmark.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2025-08-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000122, "perfPerWatt": 0.307, "tokensPerDollar": 0.086, "freshnessDays": 340, "isStale": true }, { "id": "bench-rtx4090-phi3", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 285, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Phi-3 mini at full bandwidth.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2025-08-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000059, "perfPerWatt": 0.633, "tokensPerDollar": 0.178, "freshnessDays": 332, "isStale": true }, { "id": "bench-rtx4090-gemma", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "Gemma 2 9B Q4.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2025-08-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000184, "perfPerWatt": 0.204, "tokensPerDollar": 0.058, "freshnessDays": 319, "isStale": true }, { "id": "bench-rtx4090-embed", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 5800, "quantization": "FP16", "contextLength": 512, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "amazonAsin": "B0GBCYX232", "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-09-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 309, "isStale": true }, { "id": "bench-rtx5090-mistral", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 182, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "GDDR7 advantage.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2026-02-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000116, "perfPerWatt": 0.317, "tokensPerDollar": 0.091, "freshnessDays": 145, "isStale": false }, { "id": "bench-rtx5090-phi3", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 348, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Highest single-card phi-3 result.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2026-02-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000061, "perfPerWatt": 0.605, "tokensPerDollar": 0.174, "freshnessDays": 138, "isStale": false }, { "id": "bench-rtx5090-gemma", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 122, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "Gemma 2 9B Q4 at full ctx.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2026-03-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000173, "perfPerWatt": 0.212, "tokensPerDollar": 0.061, "freshnessDays": 127, "isStale": false }, { "id": "bench-rtx5090-whisper", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 128, "quantization": "FP16", "contextLength": 0, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "faster-whisper.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2026-03-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 120, "isStale": false }, { "id": "bench-rtx5090-sdxl", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 32, "quantization": "FP16", "contextLength": 0, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "amazonAsin": "B0DQCKGDCJ", "sourceNote": "ComfyUI + TensorRT.", "sourceUrl": "https://developer.nvidia.com/blog/getting-started-with-stable-diffusion-tensorrt/", "testedAt": "2026-03-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 107, "isStale": false }, { "id": "bench-rtx4080s-mistral", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 108, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Mistral 7B fits cleanly.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2025-02-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000098, "perfPerWatt": 0.338, "tokensPerDollar": 0.108, "freshnessDays": 507, "isStale": true }, { "id": "bench-rtx4080s-gemma", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Gemma 2 9B Q4 with headroom.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2025-03-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000147, "perfPerWatt": 0.225, "tokensPerDollar": 0.072, "freshnessDays": 495, "isStale": true }, { "id": "bench-rtx4080s-phi3", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 215, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "AD103.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2025-03-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000049, "perfPerWatt": 0.672, "tokensPerDollar": 0.215, "freshnessDays": 481, "isStale": true }, { "id": "bench-rtx4080s-whisper", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 82, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "faster-whisper.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/1473", "testedAt": "2025-03-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 475, "isStale": true }, { "id": "bench-rtx4080s-embed", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 4200, "quantization": "FP16", "contextLength": 512, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-04-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 461, "isStale": true }, { "id": "bench-rx7900xtx-mistral", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 95, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "ROCm 6.2.4.", "sourceUrl": "https://github.com/ROCm/ROCm", "testedAt": "2025-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000111, "perfPerWatt": 0.268, "tokensPerDollar": 0.095, "freshnessDays": 513, "isStale": true }, { "id": "bench-rx7900xtx-phi3", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 195, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "RDNA 3 + ROCm.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1aym10w/7900_xtx_rocm_results/", "testedAt": "2025-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000054, "perfPerWatt": 0.549, "tokensPerDollar": 0.195, "freshnessDays": 489, "isStale": true }, { "id": "bench-rx7900xtx-gemma", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "Q4 fits with headroom.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2025-03-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000182, "perfPerWatt": 0.163, "tokensPerDollar": 0.058, "freshnessDays": 475, "isStale": true }, { "id": "bench-rx7900xtx-deepseek", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "Distilled R1.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-04-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000135, "perfPerWatt": 0.22, "tokensPerDollar": 0.078, "freshnessDays": 451, "isStale": true }, { "id": "bench-h100-qwen-14b", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 165, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "vLLM tensor-parallel.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-04-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001602, "perfPerWatt": 0.236, "tokensPerDollar": 0.007, "freshnessDays": 454, "isStale": true }, { "id": "bench-h100-deepseek", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 285, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "R1 distilled.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-04-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000927, "perfPerWatt": 0.407, "tokensPerDollar": 0.011, "freshnessDays": 441, "isStale": true }, { "id": "bench-h100-mistral", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 295, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Mistral 7B at max ctx.", "sourceUrl": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "testedAt": "2025-05-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000896, "perfPerWatt": 0.421, "tokensPerDollar": 0.012, "freshnessDays": 428, "isStale": true }, { "id": "bench-h100-gemma", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 235, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Gemma 2 9B.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it-GGUF", "testedAt": "2025-05-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001124, "perfPerWatt": 0.336, "tokensPerDollar": 0.009, "freshnessDays": 414, "isStale": true }, { "id": "bench-h100-phi3", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 612, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Highest phi-3 result on single GPU.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2025-06-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000432, "perfPerWatt": 0.874, "tokensPerDollar": 0.024, "freshnessDays": 401, "isStale": true }, { "id": "bench-mi300x-qwen-14b", "deviceId": "amd-mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 142, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "ROCm 6.2, vLLM.", "sourceUrl": "https://github.com/ROCm/ROCm", "testedAt": "2025-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001117, "perfPerWatt": 0.189, "tokensPerDollar": 0.009, "freshnessDays": 489, "isStale": true }, { "id": "bench-mi300x-deepseek", "deviceId": "amd-mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 248, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "Massive HBM3 bandwidth (5.3 TB/s).", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d9qjcm/mi300x_benchmarks/", "testedAt": "2025-03-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000639, "perfPerWatt": 0.331, "tokensPerDollar": 0.017, "freshnessDays": 475, "isStale": true }, { "id": "bench-mi300x-l3-70b-q8", "deviceId": "amd-mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q8", "tokensPerSec": 38, "quantization": "Q8_0", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "70B Q8 fits comfortably.", "sourceUrl": "https://github.com/ROCm/ROCm", "testedAt": "2025-04-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.004172, "perfPerWatt": 0.051, "tokensPerDollar": 0.003, "freshnessDays": 458, "isStale": true }, { "id": "bench-mi300x-embed", "deviceId": "amd-mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 14800, "quantization": "FP16", "contextLength": 512, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "TEI batch 128.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-04-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 444, "isStale": true }, { "id": "bench-m3-ultra-mistral", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 68, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 512, "tdpW": 80, "msrpUsd": 8499, "sourceNote": "M3 Ultra 819 GB/s unified.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2025-12-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001321, "perfPerWatt": 0.85, "tokensPerDollar": 0.008, "freshnessDays": 218, "isStale": true }, { "id": "bench-m3-ultra-qwen-14b", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 32768, "vramGB": 512, "tdpW": 80, "msrpUsd": 8499, "sourceNote": "32K ctx fits with 512GB unified.", "sourceUrl": "https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GGUF", "testedAt": "2025-12-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001728, "perfPerWatt": 0.65, "tokensPerDollar": 0.006, "freshnessDays": 204, "isStale": true }, { "id": "bench-m3-ultra-deepseek", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 75, "quantization": "Q4_K_M", "contextLength": 32768, "vramGB": 512, "tdpW": 80, "msrpUsd": 8499, "sourceNote": "Best ctx-length / token-rate envelope.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2026-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001198, "perfPerWatt": 0.938, "tokensPerDollar": 0.009, "freshnessDays": 183, "isStale": true }, { "id": "bench-m3-ultra-sdxl", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "sdxl-1024", "imagesPerMin": 4.2, "quantization": "FP16", "contextLength": 0, "vramGB": 512, "tdpW": 80, "msrpUsd": 8499, "sourceNote": "DrawThings MLX.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-01-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 169, "isStale": false }, { "id": "bench-m3-ultra-embed", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "embedding-bge-large", "embeddingsPerSec": 2680, "quantization": "FP16", "contextLength": 512, "vramGB": 512, "tdpW": 80, "msrpUsd": 8499, "sourceNote": "MLX TEI.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2026-02-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 152, "isStale": false }, { "id": "bench-groq-l3-8b-fp16-v2", "deviceId": "groq-lpu", "deviceName": "Groq LPU (cloud)", "brand": "Groq", "deviceClass": "asic", "workload": "llama3-8b-fp16", "tokensPerSec": 1850, "quantization": "FP16", "contextLength": 8192, "vramGB": 230, "tdpW": 350, "msrpUsd": 20000, "sourceNote": "Vendor benchmark, single sequence.", "sourceUrl": "https://groq.com/docs/", "testedAt": "2026-04-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000114, "perfPerWatt": 5.286, "tokensPerDollar": 0.093, "freshnessDays": 81, "isStale": false }, { "id": "bench-groq-mistral", "deviceId": "groq-lpu", "deviceName": "Groq LPU (cloud)", "brand": "Groq", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 1280, "quantization": "FP16", "contextLength": 8192, "vramGB": 230, "tdpW": 350, "msrpUsd": 20000, "sourceNote": "Groq vendor numbers.", "sourceUrl": "https://groq.com/docs/", "testedAt": "2026-05-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000165, "perfPerWatt": 3.657, "tokensPerDollar": 0.064, "freshnessDays": 67, "isStale": false }, { "id": "bench-trainium2-l3-8b-q4", "deviceId": "aws-trainium2", "deviceName": "AWS Trainium2", "brand": "AWS", "deviceClass": "asic", "workload": "llama3-8b-q4", "tokensPerSec": 215, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 500, "msrpUsd": 28000, "sourceNote": "AWS Neuron SDK 2.20.", "sourceUrl": "https://aws.amazon.com/machine-learning/trainium/", "testedAt": "2026-03-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001377, "perfPerWatt": 0.43, "tokensPerDollar": 0.008, "freshnessDays": 117, "isStale": false }, { "id": "bench-trainium2-l3-70b-q4", "deviceId": "aws-trainium2", "deviceName": "AWS Trainium2", "brand": "AWS", "deviceClass": "asic", "workload": "llama3-70b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 500, "msrpUsd": 28000, "sourceNote": "Two-chip pod.", "sourceUrl": "https://aws.amazon.com/blogs/aws/aws-trainium2-instances/", "testedAt": "2026-03-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.005103, "perfPerWatt": 0.116, "tokensPerDollar": 0.002, "freshnessDays": 104, "isStale": false }, { "id": "bench-orin-agx-64gb-qwen", "deviceId": "jetson-orin-agx-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "qwen2.5-14b-q4", "tokensPerSec": 16, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "JetPack 6.0.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-08-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001321, "perfPerWatt": 0.267, "tokensPerDollar": 0.008, "freshnessDays": 332, "isStale": true }, { "id": "bench-orin-agx-64gb-deepseek", "deviceId": "jetson-orin-agx-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "R1 distilled on edge.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-09-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00096, "perfPerWatt": 0.367, "tokensPerDollar": 0.011, "freshnessDays": 309, "isStale": true }, { "id": "bench-orin-agx-64gb-sdxl", "deviceId": "jetson-orin-agx-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "sdxl-1024", "imagesPerMin": 1.8, "quantization": "FP16", "contextLength": 0, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "Stable Diffusion on edge.", "sourceUrl": "https://www.jetson-ai-lab.com/tutorial_stable-diffusion.html", "testedAt": "2025-09-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 295, "isStale": true }, { "id": "bench-orin-nx-l3-8b-q4", "deviceId": "jetson-orin-nx-16gb", "deviceName": "NVIDIA Jetson Orin NX 16GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "llama3-8b-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 25, "msrpUsd": 599, "sourceNote": "Edge SOM, mid-tier.", "sourceUrl": "https://www.jetson-ai-lab.com/benchmarks.html", "testedAt": "2025-10-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000352, "perfPerWatt": 0.72, "tokensPerDollar": 0.03, "freshnessDays": 279, "isStale": true }, { "id": "bench-orin-nx-phi3", "deviceId": "jetson-orin-nx-16gb", "deviceName": "NVIDIA Jetson Orin NX 16GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 25, "msrpUsd": 599, "sourceNote": "Phi-3 in 25W.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2025-10-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000167, "perfPerWatt": 1.52, "tokensPerDollar": 0.063, "freshnessDays": 268, "isStale": true }, { "id": "bench-rpi5-phi3", "deviceId": "raspberry-pi-5", "deviceName": "Raspberry Pi 5 8GB", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 4.2, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 0, "tdpW": 12, "msrpUsd": 80, "sourceNote": "ARM Cortex-A76, llama.cpp CPU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/4167", "testedAt": "2025-06-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000201, "perfPerWatt": 0.35, "tokensPerDollar": 0.053, "freshnessDays": 383, "isStale": true }, { "id": "bench-rpi5-mistral", "deviceId": "raspberry-pi-5", "deviceName": "Raspberry Pi 5 8GB", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "mistral-7b-q4", "tokensPerSec": 1.8, "quantization": "Q4_K_M", "contextLength": 2048, "vramGB": 0, "tdpW": 12, "msrpUsd": 80, "sourceNote": "Slow but functional — proves local LLM on a $80 board.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1d8mqg7/raspberry_pi_5_mistral_7b/", "testedAt": "2025-07-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00047, "perfPerWatt": 0.15, "tokensPerDollar": 0.023, "freshnessDays": 371, "isStale": true }, { "id": "bench-lunar-lake-phi3", "deviceId": "intel-core-ultra-258v", "deviceName": "Intel Core Ultra 7 258V (Lunar Lake)", "brand": "Intel", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 22, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 17, "msrpUsd": 1299, "sourceNote": "48 TOPS NPU 4, OpenVINO.", "sourceUrl": "https://www.intel.com/content/www/us/en/products/sku/240777/intel-core-ultra-7-processor-258v/specifications.html", "testedAt": "2025-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000624, "perfPerWatt": 1.294, "tokensPerDollar": 0.017, "freshnessDays": 548, "isStale": true }, { "id": "bench-lunar-lake-whisper", "deviceId": "intel-core-ultra-258v", "deviceName": "Intel Core Ultra 7 258V (Lunar Lake)", "brand": "Intel", "deviceClass": "npu", "workload": "whisper-large-v3", "audioRtfx": 4.2, "quantization": "INT8", "contextLength": 0, "vramGB": 0, "tdpW": 17, "msrpUsd": 1299, "sourceNote": "OpenVINO + NPU.", "sourceUrl": "https://github.com/openvinotoolkit/openvino_notebooks", "testedAt": "2025-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 521, "isStale": true }, { "id": "bench-lunar-lake-l3-8b", "deviceId": "intel-core-ultra-258v", "deviceName": "Intel Core Ultra 7 258V (Lunar Lake)", "brand": "Intel", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 5.5, "quantization": "INT4", "contextLength": 2048, "vramGB": 0, "tdpW": 17, "msrpUsd": 1299, "sourceNote": "Hybrid NPU+iGPU via OpenVINO.", "sourceUrl": "https://github.com/intel/openvino-genai", "testedAt": "2025-03-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002496, "perfPerWatt": 0.324, "tokensPerDollar": 0.004, "freshnessDays": 493, "isStale": true }, { "id": "bench-sdx-elite-l3-8b", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite (X1E-84-100)", "brand": "Qualcomm", "deviceClass": "npu", "workload": "llama3-8b-q4", "tokensPerSec": 12, "quantization": "INT4", "contextLength": 4096, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "amazonAsin": "B0CXL8T7GC", "sourceNote": "QNN SDK, 45 TOPS Hexagon NPU.", "sourceUrl": "https://github.com/quic/ai-hub-models", "testedAt": "2025-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001056, "perfPerWatt": 0.522, "tokensPerDollar": 0.01, "freshnessDays": 513, "isStale": true }, { "id": "bench-sdx-elite-phi3", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite (X1E-84-100)", "brand": "Qualcomm", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 28, "quantization": "INT4", "contextLength": 4096, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "amazonAsin": "B0CXL8T7GC", "sourceNote": "Vendor-optimized Phi-3.", "sourceUrl": "https://github.com/microsoft/Phi-3CookBook", "testedAt": "2025-03-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000453, "perfPerWatt": 1.217, "tokensPerDollar": 0.023, "freshnessDays": 482, "isStale": true }, { "id": "bench-rtx5080-fresh", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 7200, "quantization": "FP16", "contextLength": 512, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Re-bench: TEI 1.5 + driver R570 tuning.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2026-05-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 48, "isStale": false }, { "id": "bench-strix-halo-fresh", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "Fresh DeepSeek R1 run with ROCm 6.3.", "sourceUrl": "https://github.com/ROCm/ROCm", "testedAt": "2026-05-24", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000968, "perfPerWatt": 0.2, "tokensPerDollar": 0.011, "freshnessDays": 47, "isStale": false }, { "id": "bench-dgx-spark-fresh", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 82, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "Re-bench at 16K ctx — Grace bandwidth wins.", "sourceUrl": "https://www.nvidia.com/en-us/project-digits/", "testedAt": "2026-05-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000387, "perfPerWatt": 0.342, "tokensPerDollar": 0.027, "freshnessDays": 46, "isStale": false }, { "id": "bench-mac-mini-m4pro-fresh", "deviceId": "mac-mini-m4-pro-48gb", "deviceName": "Apple Mac mini M4 Pro 48GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 16384, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "MLX 0.21 + macOS 15.4.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2026-05-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00066, "perfPerWatt": 0.914, "tokensPerDollar": 0.016, "freshnessDays": 45, "isStale": false }, { "id": "bench-x3-rtx5090-mistral7b", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 178, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Mistral 7B Q4 with llama.cpp b3700 on CUDA 12.8.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/8721", "testedAt": "2025-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000119, "perfPerWatt": 0.31, "tokensPerDollar": 0.089, "freshnessDays": 521, "isStale": true }, { "id": "bench-x3-rtx5090-gemma9b", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 152, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Gemma 2 9B Q4 — bandwidth-bound batch 1.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2025-02-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000139, "perfPerWatt": 0.264, "tokensPerDollar": 0.076, "freshnessDays": 507, "isStale": true }, { "id": "bench-x3-rtx5090-l3-8b-32k", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 162, "quantization": "Q4_K_M", "contextLength": 32768, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Extended 32k context — KV cache cost reduces tok/s ~10%.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9000", "testedAt": "2025-03-06", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00013, "perfPerWatt": 0.282, "tokensPerDollar": 0.081, "freshnessDays": 491, "isStale": true }, { "id": "bench-x3-rtx5090-l3-8b-batch8", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 720, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Batch 8 throughput — aggregate tok/s across streams.", "sourceUrl": "https://github.com/vllm-project/vllm/pull/6789", "testedAt": "2025-03-12", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000029, "perfPerWatt": 1.252, "tokensPerDollar": 0.36, "freshnessDays": 485, "isStale": true }, { "id": "bench-x3-rtx5090-whisper", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 78, "quantization": "FP16", "contextLength": 0, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "faster-whisper, batch 1 on Blackwell.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/issues/911", "testedAt": "2025-03-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 477, "isStale": true }, { "id": "bench-x3-rtx5090-embed", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 12200, "quantization": "FP16", "contextLength": 512, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "TEI 1.5, batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-04-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 464, "isStale": true }, { "id": "bench-x3-rtx4090-mistral7b", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 128, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "llama.cpp b3500 CUDA 12.4.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4321", "testedAt": "2024-10-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000132, "perfPerWatt": 0.284, "tokensPerDollar": 0.08, "freshnessDays": 634, "isStale": true }, { "id": "bench-x3-rtx4090-gemma9b", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 108, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "Gemma 2 9B Q4 via Ollama 0.3.6.", "sourceUrl": "https://github.com/ollama/ollama/issues/6014", "testedAt": "2024-11-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000156, "perfPerWatt": 0.24, "tokensPerDollar": 0.068, "freshnessDays": 615, "isStale": true }, { "id": "bench-x3-rtx4090-qwen14b", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "Qwen 2.5 14B Q4_K_M with llama.cpp b3500.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/4321", "testedAt": "2024-11-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000217, "perfPerWatt": 0.173, "tokensPerDollar": 0.049, "freshnessDays": 598, "isStale": true }, { "id": "bench-x3-rtx4090-l3-8b-32k", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 112, "quantization": "Q4_K_M", "contextLength": 32768, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "32k context — ~10% throughput penalty vs 4k baseline.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1cyz3a4/long_context_4090_test/", "testedAt": "2024-09-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000151, "perfPerWatt": 0.249, "tokensPerDollar": 0.07, "freshnessDays": 670, "isStale": true }, { "id": "bench-x3-rtx4090-l3-8b-batch8", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 540, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "vLLM 0.5.2 batch 8 aggregate.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/4567", "testedAt": "2024-08-21", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000031, "perfPerWatt": 1.2, "tokensPerDollar": 0.338, "freshnessDays": 688, "isStale": true }, { "id": "bench-x3-rtx4090-embed", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 8500, "quantization": "FP16", "contextLength": 512, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "TEI 1.5 batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-07-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 725, "isStale": true }, { "id": "bench-x3-rtx4090-phi3", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 245, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "Phi-3 Mini Q4, llama.cpp.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-06-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000069, "perfPerWatt": 0.544, "tokensPerDollar": 0.153, "freshnessDays": 740, "isStale": true }, { "id": "bench-x3-h100-mistral7b", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 215, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "vLLM 0.5.4 single-stream.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2024-05-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001229, "perfPerWatt": 0.307, "tokensPerDollar": 0.009, "freshnessDays": 789, "isStale": true }, { "id": "bench-x3-h100-gemma9b", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 188, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "TRT-LLM 0.10.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2024-09-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001406, "perfPerWatt": 0.269, "tokensPerDollar": 0.008, "freshnessDays": 653, "isStale": true }, { "id": "bench-x3-h100-qwen14b", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 145, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Qwen 2.5 14B Q4 vLLM.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/8123", "testedAt": "2024-10-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001822, "perfPerWatt": 0.207, "tokensPerDollar": 0.006, "freshnessDays": 639, "isStale": true }, { "id": "bench-x3-h100-l3-70b-batch8", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 410, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Batch 8 aggregate — TRT-LLM continuous batching.", "sourceUrl": "https://mlcommons.org/benchmarks/inference-datacenter/", "testedAt": "2024-08-30", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000645, "perfPerWatt": 0.586, "tokensPerDollar": 0.016, "freshnessDays": 679, "isStale": true }, { "id": "bench-x3-h100-l3-8b-128k", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 178, "quantization": "Q4_K_M", "contextLength": 131072, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "128k context — KV cache cost dominates.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/5512", "testedAt": "2024-11-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001485, "perfPerWatt": 0.254, "tokensPerDollar": 0.007, "freshnessDays": 609, "isStale": true }, { "id": "bench-x3-h100-deepseek7b", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 198, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "DeepSeek-R1-Distill-Qwen-7B on H100.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-01-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001335, "perfPerWatt": 0.283, "tokensPerDollar": 0.008, "freshnessDays": 531, "isStale": true }, { "id": "bench-x3-m4max-mistral7b", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 64, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "MLX 0.20 on macOS 15.2.", "sourceUrl": "https://github.com/ml-explore/mlx/discussions/1244", "testedAt": "2024-12-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000776, "perfPerWatt": 0.985, "tokensPerDollar": 0.014, "freshnessDays": 579, "isStale": true }, { "id": "bench-x3-m4max-gemma9b", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "Gemma 2 9B Q4 via MLX.", "sourceUrl": "https://huggingface.co/mlx-community/gemma-2-9b-it-4bit", "testedAt": "2024-12-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001035, "perfPerWatt": 0.738, "tokensPerDollar": 0.01, "freshnessDays": 567, "isStale": true }, { "id": "bench-x3-m4max-qwen14b", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 34, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "Qwen 2.5 14B MLX 4-bit.", "sourceUrl": "https://huggingface.co/mlx-community/Qwen2.5-14B-Instruct-4bit", "testedAt": "2025-01-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001461, "perfPerWatt": 0.523, "tokensPerDollar": 0.007, "freshnessDays": 547, "isStale": true }, { "id": "bench-x3-m4max-l3-8b-32k", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 32768, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "32k context on M4 Max — KV in unified memory.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1g8jq8e/m4_max_long_context/", "testedAt": "2025-01-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001035, "perfPerWatt": 0.738, "tokensPerDollar": 0.01, "freshnessDays": 534, "isStale": true }, { "id": "bench-x3-m4max-whisper", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "whisper-large-v3", "audioRtfx": 12.5, "quantization": "FP16", "contextLength": 0, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "whisper.cpp Metal backend.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/2113", "testedAt": "2025-02-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 515, "isStale": true }, { "id": "bench-x3-m4max-embed", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "embedding-bge-large", "embeddingsPerSec": 3500, "quantization": "FP16", "contextLength": 512, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "MLX text embeddings, batch 32.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2025-02-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 503, "isStale": true }, { "id": "bench-x3-rx7900xtx-mistral7b", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 88, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "ROCm 6.2 llama.cpp HIP build.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/7593", "testedAt": "2024-08-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00012, "perfPerWatt": 0.248, "tokensPerDollar": 0.088, "freshnessDays": 694, "isStale": true }, { "id": "bench-x3-rx7900xtx-gemma9b", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "ROCm 6.2 HIP.", "sourceUrl": "https://github.com/ROCm/ROCm/discussions/2986", "testedAt": "2024-09-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000147, "perfPerWatt": 0.203, "tokensPerDollar": 0.072, "freshnessDays": 673, "isStale": true }, { "id": "bench-x3-rx7900xtx-qwen14b", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "Qwen 2.5 14B Q4 on ROCm 6.2.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9ax2h/7900_xtx_qwen_25/", "testedAt": "2024-10-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000203, "perfPerWatt": 0.146, "tokensPerDollar": 0.052, "freshnessDays": 637, "isStale": true }, { "id": "bench-x3-rx7900xtx-embed", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 5800, "quantization": "FP16", "contextLength": 512, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "TEI on ROCm 6.2.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference/issues/322", "testedAt": "2024-11-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 613, "isStale": true }, { "id": "bench-x3-rtx4080s-mistral7b", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 102, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "llama.cpp CUDA build.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/issues/6543", "testedAt": "2024-07-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000104, "perfPerWatt": 0.319, "tokensPerDollar": 0.102, "freshnessDays": 718, "isStale": true }, { "id": "bench-x3-rtx4080s-gemma9b", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 86, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Gemma 2 9B Q4.", "sourceUrl": "https://github.com/ollama/ollama/issues/6014", "testedAt": "2024-08-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000123, "perfPerWatt": 0.269, "tokensPerDollar": 0.086, "freshnessDays": 700, "isStale": true }, { "id": "bench-x3-rtx4080s-deepseek7b", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 95, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "DeepSeek-R1 Distill 7B.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-02-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000111, "perfPerWatt": 0.297, "tokensPerDollar": 0.095, "freshnessDays": 524, "isStale": true }, { "id": "bench-x3-rtx4080s-embed", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 6900, "quantization": "FP16", "contextLength": 512, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "TEI batch 32.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-09-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 662, "isStale": true }, { "id": "bench-x3-rtx5080-mistral7b", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 122, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Blackwell consumer.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1iax2pd/rtx_5080_local_llm/", "testedAt": "2025-02-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000087, "perfPerWatt": 0.339, "tokensPerDollar": 0.122, "freshnessDays": 510, "isStale": true }, { "id": "bench-x3-rtx5080-gemma9b", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 105, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Gemma 2 9B on Blackwell.", "sourceUrl": "https://github.com/ollama/ollama/issues/8421", "testedAt": "2025-02-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000101, "perfPerWatt": 0.292, "tokensPerDollar": 0.105, "freshnessDays": 500, "isStale": true }, { "id": "bench-x3-rtx5080-embed", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 8200, "quantization": "FP16", "contextLength": 512, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "TEI on Blackwell.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-03-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 493, "isStale": true }, { "id": "bench-x3-mi300x-mistral7b", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 195, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "vLLM ROCm 6.2.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/4756", "testedAt": "2024-07-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000813, "perfPerWatt": 0.26, "tokensPerDollar": 0.013, "freshnessDays": 732, "isStale": true }, { "id": "bench-x3-mi300x-gemma9b", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 168, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "ROCm 6.2 HIP.", "sourceUrl": "https://www.amd.com/en/products/accelerators/instinct/mi300/mi300x.html", "testedAt": "2024-08-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000944, "perfPerWatt": 0.224, "tokensPerDollar": 0.011, "freshnessDays": 695, "isStale": true }, { "id": "bench-x3-mi300x-qwen14b", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 128, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "Qwen 2.5 14B on MI300X.", "sourceUrl": "https://github.com/ROCm/ROCm/discussions/3214", "testedAt": "2024-10-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001239, "perfPerWatt": 0.171, "tokensPerDollar": 0.009, "freshnessDays": 629, "isStale": true }, { "id": "bench-x3-mi300x-l3-70b-batch8", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 380, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "vLLM batch 8 aggregate.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/4756", "testedAt": "2024-09-12", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000417, "perfPerWatt": 0.507, "tokensPerDollar": 0.025, "freshnessDays": 666, "isStale": true }, { "id": "bench-x3-rtx3090-mistral7b", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Ampere — older but still common.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1akhd7w/rtx_3090_mistral_7b/", "testedAt": "2024-04-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000203, "perfPerWatt": 0.223, "tokensPerDollar": 0.052, "freshnessDays": 809, "isStale": true }, { "id": "bench-x3-rtx3090-gemma9b", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 65, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Gemma 2 9B Q4 on Ampere.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/8201", "testedAt": "2024-06-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000244, "perfPerWatt": 0.186, "tokensPerDollar": 0.043, "freshnessDays": 758, "isStale": true }, { "id": "bench-x3-rtx3090-qwen14b", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Qwen 2.5 14B on 3090.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9p2dx/qwen_25_3090/", "testedAt": "2024-10-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00033, "perfPerWatt": 0.137, "tokensPerDollar": 0.032, "freshnessDays": 623, "isStale": true }, { "id": "bench-x3-rtx3090-embed", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 4900, "quantization": "FP16", "contextLength": 512, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "TEI Ampere.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-05-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 783, "isStale": true }, { "id": "bench-x3-rtx5070-mistral7b", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 88, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Blackwell mid-range.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1ip6q5w/5070_first_test/", "testedAt": "2025-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000066, "perfPerWatt": 0.352, "tokensPerDollar": 0.16, "freshnessDays": 489, "isStale": true }, { "id": "bench-x3-rtx5070-gemma9b", "deviceId": "rtx-5070", "deviceName": "NVIDIA GeForce RTX 5070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 76, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 250, "msrpUsd": 549, "sourceNote": "Gemma 2 9B Q4 tight VRAM fit.", "sourceUrl": "https://github.com/ollama/ollama/issues/8421", "testedAt": "2025-03-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000076, "perfPerWatt": 0.304, "tokensPerDollar": 0.138, "freshnessDays": 482, "isStale": true }, { "id": "bench-x3-rtx5070ti-mistral7b", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 108, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "Blackwell mid-high.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9521", "testedAt": "2025-02-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000073, "perfPerWatt": 0.36, "tokensPerDollar": 0.144, "freshnessDays": 497, "isStale": true }, { "id": "bench-x3-rtx5070ti-gemma9b", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "Gemma 2 9B Q4 on 5070 Ti.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2025-03-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000086, "perfPerWatt": 0.307, "tokensPerDollar": 0.123, "freshnessDays": 475, "isStale": true }, { "id": "bench-x3-rtx5070ti-embed", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 7100, "quantization": "FP16", "contextLength": 512, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "TEI Blackwell.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-04-03", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 463, "isStale": true }, { "id": "bench-x3-arcb580-mistral7b", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "IPEX-LLM on Arc Battlemage.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm/issues/12421", "testedAt": "2025-01-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000063, "perfPerWatt": 0.221, "tokensPerDollar": 0.169, "freshnessDays": 542, "isStale": true }, { "id": "bench-x3-arcb580-gemma9b", "deviceId": "arc-b580", "deviceName": "Intel Arc B580 12GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 35, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 190, "msrpUsd": 249, "sourceNote": "IPEX-LLM Gemma 2 9B.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2025-01-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000075, "perfPerWatt": 0.184, "tokensPerDollar": 0.141, "freshnessDays": 528, "isStale": true }, { "id": "bench-x3-groq-mistral7b", "deviceId": "groq-lpu", "deviceName": "Groq LPU Inference Engine", "brand": "Groq", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 750, "quantization": "FP8", "contextLength": 4096, "vramGB": 230, "tdpW": 215, "msrpUsd": 20000, "sourceNote": "Groq LPU dedicated transformer engine.", "sourceUrl": "https://groq.com/docs/api-reference/", "testedAt": "2024-12-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000282, "perfPerWatt": 3.488, "tokensPerDollar": 0.038, "freshnessDays": 583, "isStale": true }, { "id": "bench-x3-groq-gemma9b", "deviceId": "groq-lpu", "deviceName": "Groq LPU Inference Engine", "brand": "Groq", "deviceClass": "asic", "workload": "gemma-2-9b-q4", "tokensPerSec": 580, "quantization": "FP8", "contextLength": 8192, "vramGB": 230, "tdpW": 215, "msrpUsd": 20000, "sourceNote": "Groq Gemma 2 9B-IT.", "sourceUrl": "https://groq.com/llamaindex/", "testedAt": "2024-09-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000364, "perfPerWatt": 2.698, "tokensPerDollar": 0.029, "freshnessDays": 658, "isStale": true }, { "id": "bench-x3-b200-mistral7b", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 415, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Blackwell DC FP4 not used.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001019, "perfPerWatt": 0.415, "tokensPerDollar": 0.01, "freshnessDays": 548, "isStale": true }, { "id": "bench-x3-b200-gemma9b", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 332, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Gemma 2 9B on B200.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001273, "perfPerWatt": 0.332, "tokensPerDollar": 0.008, "freshnessDays": 513, "isStale": true }, { "id": "bench-x3-b200-qwen14b", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 268, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Qwen 2.5 14B on B200.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-03-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001578, "perfPerWatt": 0.268, "tokensPerDollar": 0.007, "freshnessDays": 496, "isStale": true }, { "id": "bench-x3-h200-mistral7b", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 268, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "H200 HBM3e bandwidth.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2024-05-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001183, "perfPerWatt": 0.383, "tokensPerDollar": 0.009, "freshnessDays": 797, "isStale": true }, { "id": "bench-x3-h200-gemma9b", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 220, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Gemma 2 9B on H200.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2024-07-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001441, "perfPerWatt": 0.314, "tokensPerDollar": 0.007, "freshnessDays": 710, "isStale": true }, { "id": "bench-x3-h200-qwen14b", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 178, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Qwen 2.5 14B on H200.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/8123", "testedAt": "2024-12-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001781, "perfPerWatt": 0.254, "tokensPerDollar": 0.006, "freshnessDays": 572, "isStale": true }, { "id": "bench-x3-m3ultra-mistral7b", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 82, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 512, "tdpW": 100, "msrpUsd": 9999, "sourceNote": "Mac Pro M3 Ultra MLX.", "sourceUrl": "https://www.apple.com/mac-pro/specs/", "testedAt": "2024-06-24", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001289, "perfPerWatt": 0.82, "tokensPerDollar": 0.008, "freshnessDays": 746, "isStale": true }, { "id": "bench-x3-m3ultra-gemma9b", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 62, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 512, "tdpW": 100, "msrpUsd": 9999, "sourceNote": "Gemma 2 9B MLX.", "sourceUrl": "https://huggingface.co/mlx-community/gemma-2-9b-it-4bit", "testedAt": "2024-08-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001705, "perfPerWatt": 0.62, "tokensPerDollar": 0.006, "freshnessDays": 707, "isStale": true }, { "id": "bench-x3-m3ultra-qwen14b", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 44, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 512, "tdpW": 100, "msrpUsd": 9999, "sourceNote": "Qwen 2.5 14B MLX 4-bit.", "sourceUrl": "https://huggingface.co/mlx-community/Qwen2.5-14B-Instruct-4bit", "testedAt": "2024-10-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002402, "perfPerWatt": 0.44, "tokensPerDollar": 0.004, "freshnessDays": 630, "isStale": true }, { "id": "bench-x3-m2ultra-mistral7b", "deviceId": "apple-m2-ultra", "deviceName": "Apple M2 Ultra (76c GPU, 192GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 60, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 192, "tdpW": 80, "msrpUsd": 6999, "sourceNote": "Mac Studio M2 Ultra MLX 0.18.", "sourceUrl": "https://github.com/ml-explore/mlx/discussions/1244", "testedAt": "2024-04-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001233, "perfPerWatt": 0.75, "tokensPerDollar": 0.009, "freshnessDays": 822, "isStale": true }, { "id": "bench-x3-m2ultra-gemma9b", "deviceId": "apple-m2-ultra", "deviceName": "Apple M2 Ultra (76c GPU, 192GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 44, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 80, "msrpUsd": 6999, "sourceNote": "Gemma 2 9B MLX.", "sourceUrl": "https://huggingface.co/mlx-community/gemma-2-9b-it-4bit", "testedAt": "2024-07-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001681, "perfPerWatt": 0.55, "tokensPerDollar": 0.006, "freshnessDays": 728, "isStale": true }, { "id": "bench-x3-m2ultra-qwen14b", "deviceId": "apple-m2-ultra", "deviceName": "Apple M2 Ultra (76c GPU, 192GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "qwen2.5-14b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 80, "msrpUsd": 6999, "sourceNote": "Qwen 2.5 14B 4-bit MLX.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-10-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002312, "perfPerWatt": 0.4, "tokensPerDollar": 0.005, "freshnessDays": 644, "isStale": true }, { "id": "bench-x3-dgx-spark-mistral7b", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 145, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "Grace-Blackwell unified memory.", "sourceUrl": "https://www.nvidia.com/en-us/project-digits/", "testedAt": "2025-04-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000219, "perfPerWatt": 0.604, "tokensPerDollar": 0.048, "freshnessDays": 458, "isStale": true }, { "id": "bench-x3-dgx-spark-gemma9b", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 118, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "DGX Spark Gemma 2 9B.", "sourceUrl": "https://www.nvidia.com/en-us/project-digits/", "testedAt": "2025-04-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000269, "perfPerWatt": 0.492, "tokensPerDollar": 0.039, "freshnessDays": 451, "isStale": true }, { "id": "bench-x3-strix-halo-mistral7b", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "Strix Halo iGPU + NPU.", "sourceUrl": "https://www.amd.com/en/products/processors/laptop/ryzen.html", "testedAt": "2025-01-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000612, "perfPerWatt": 0.317, "tokensPerDollar": 0.017, "freshnessDays": 526, "isStale": true }, { "id": "bench-x3-strix-halo-gemma9b", "deviceId": "amd-ryzen-ai-max-plus-395", "deviceName": "AMD Ryzen AI Max+ 395 (Strix Halo, 96GB)", "brand": "AMD", "deviceClass": "npu", "workload": "gemma-2-9b-q4", "tokensPerSec": 30, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 120, "msrpUsd": 2199, "sourceNote": "Gemma 2 9B on Strix Halo.", "sourceUrl": "https://github.com/ROCm/ROCm/discussions/3320", "testedAt": "2025-02-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000775, "perfPerWatt": 0.25, "tokensPerDollar": 0.014, "freshnessDays": 511, "isStale": true }, { "id": "bench-x3-x-elite-mistral7b", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite X1E-84-100", "brand": "Qualcomm", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "sourceNote": "Hexagon NPU via ONNX Runtime.", "sourceUrl": "https://www.qualcomm.com/products/mobile-compute/snapdragon-x-elite", "testedAt": "2024-08-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000576, "perfPerWatt": 0.957, "tokensPerDollar": 0.018, "freshnessDays": 683, "isStale": true }, { "id": "bench-x3-x-elite-phi3", "deviceId": "qualcomm-snapdragon-x1e-84-100", "deviceName": "Qualcomm Snapdragon X Elite X1E-84-100", "brand": "Qualcomm", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 23, "msrpUsd": 1199, "sourceNote": "Phi-3 Mini via QNN.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-onnx", "testedAt": "2024-09-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000334, "perfPerWatt": 1.652, "tokensPerDollar": 0.032, "freshnessDays": 648, "isStale": true }, { "id": "bench-x3-l40s-mistral7b", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 142, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "L40S vLLM 0.5.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/4567", "testedAt": "2024-06-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000581, "perfPerWatt": 0.406, "tokensPerDollar": 0.018, "freshnessDays": 752, "isStale": true }, { "id": "bench-x3-l40s-gemma9b", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 118, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Gemma 2 9B L40S.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2024-08-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000699, "perfPerWatt": 0.337, "tokensPerDollar": 0.015, "freshnessDays": 686, "isStale": true }, { "id": "bench-x3-l40s-qwen14b", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Qwen 2.5 14B L40S.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/8123", "testedAt": "2024-11-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000896, "perfPerWatt": 0.263, "tokensPerDollar": 0.012, "freshnessDays": 602, "isStale": true }, { "id": "bench-x3-a100-80-mistral7b", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 SXM4 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 168, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "Ampere DC.", "sourceUrl": "https://mlcommons.org/benchmarks/inference-datacenter/", "testedAt": "2024-04-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000944, "perfPerWatt": 0.42, "tokensPerDollar": 0.011, "freshnessDays": 816, "isStale": true }, { "id": "bench-x3-a100-80-gemma9b", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 SXM4 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "Gemma 2 9B A100.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2024-07-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001149, "perfPerWatt": 0.345, "tokensPerDollar": 0.009, "freshnessDays": 731, "isStale": true }, { "id": "bench-x3-a100-80-qwen14b", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 SXM4 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 108, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "Qwen 2.5 14B A100.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/4567", "testedAt": "2024-10-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001468, "perfPerWatt": 0.27, "tokensPerDollar": 0.007, "freshnessDays": 618, "isStale": true }, { "id": "bench-x3-a100-40-mistral7b", "deviceId": "a100-40gb", "deviceName": "NVIDIA A100 SXM4 40GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 162, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 40, "tdpW": 400, "msrpUsd": 10000, "sourceNote": "Ampere DC 40GB variant.", "sourceUrl": "https://mlcommons.org/benchmarks/inference-datacenter/", "testedAt": "2024-04-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000652, "perfPerWatt": 0.405, "tokensPerDollar": 0.016, "freshnessDays": 809, "isStale": true }, { "id": "bench-x3-a100-40-gemma9b", "deviceId": "a100-40gb", "deviceName": "NVIDIA A100 SXM4 40GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 130, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 40, "tdpW": 400, "msrpUsd": 10000, "sourceNote": "Gemma 2 9B A100 40GB.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2024-07-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000813, "perfPerWatt": 0.325, "tokensPerDollar": 0.013, "freshnessDays": 720, "isStale": true }, { "id": "bench-x3-rtx6000ada-mistral7b", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 125, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "Ada Lovelace pro.", "sourceUrl": "https://www.nvidia.com/en-us/design-visualization/rtx-6000/", "testedAt": "2024-05-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000575, "perfPerWatt": 0.417, "tokensPerDollar": 0.018, "freshnessDays": 772, "isStale": true }, { "id": "bench-x3-rtx6000ada-gemma9b", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 104, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "Gemma 2 9B RTX 6000 Ada.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2024-08-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000691, "perfPerWatt": 0.347, "tokensPerDollar": 0.015, "freshnessDays": 698, "isStale": true }, { "id": "bench-x3-rtx6000ada-qwen14b", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 76, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "Qwen 2.5 14B Ada pro.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9000", "testedAt": "2024-10-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000946, "perfPerWatt": 0.253, "tokensPerDollar": 0.011, "freshnessDays": 626, "isStale": true }, { "id": "bench-x3-rtx4070-mistral7b", "deviceId": "rtx-4070", "deviceName": "NVIDIA GeForce RTX 4070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 76, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 200, "msrpUsd": 599, "sourceNote": "Ada midrange.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1bj4u5n/4070_mistral_test/", "testedAt": "2024-05-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000083, "perfPerWatt": 0.38, "tokensPerDollar": 0.127, "freshnessDays": 797, "isStale": true }, { "id": "bench-x3-rtx4070-gemma9b", "deviceId": "rtx-4070", "deviceName": "NVIDIA GeForce RTX 4070 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 64, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 200, "msrpUsd": 599, "sourceNote": "Gemma 2 9B fits Q4 on 12GB.", "sourceUrl": "https://github.com/ollama/ollama/issues/6014", "testedAt": "2024-07-17", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000099, "perfPerWatt": 0.32, "tokensPerDollar": 0.107, "freshnessDays": 723, "isStale": true }, { "id": "bench-x3-rtx4070ti-mistral7b", "deviceId": "rtx-4070-ti", "deviceName": "NVIDIA GeForce RTX 4070 Ti 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 89, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 285, "msrpUsd": 799, "sourceNote": "Ada 4070 Ti.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/8201", "testedAt": "2024-06-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000095, "perfPerWatt": 0.312, "tokensPerDollar": 0.111, "freshnessDays": 762, "isStale": true }, { "id": "bench-x3-rtx4070ti-gemma9b", "deviceId": "rtx-4070-ti", "deviceName": "NVIDIA GeForce RTX 4070 Ti 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 75, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 285, "msrpUsd": 799, "sourceNote": "Gemma 2 9B Q4 4070 Ti.", "sourceUrl": "https://huggingface.co/google/gemma-2-9b-it", "testedAt": "2024-08-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000113, "perfPerWatt": 0.263, "tokensPerDollar": 0.094, "freshnessDays": 691, "isStale": true }, { "id": "bench-x3-rtx4060-mistral7b", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Ada entry 8GB.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1bm0a2n/rtx_4060_local_llm/", "testedAt": "2024-05-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000075, "perfPerWatt": 0.365, "tokensPerDollar": 0.14, "freshnessDays": 787, "isStale": true }, { "id": "bench-x3-rtx4060-phi3", "deviceId": "rtx-4060", "deviceName": "NVIDIA GeForce RTX 4060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 95, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 115, "msrpUsd": 299, "sourceNote": "Phi-3 Mini Q4 fits comfortably.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-06-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000033, "perfPerWatt": 0.826, "tokensPerDollar": 0.318, "freshnessDays": 744, "isStale": true }, { "id": "bench-x3-rtx4060ti16-mistral7b", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 56, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "16GB VRAM sweet-spot.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1bw2x9k/4060_ti_16gb_for_llm/", "testedAt": "2024-04-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000094, "perfPerWatt": 0.339, "tokensPerDollar": 0.112, "freshnessDays": 801, "isStale": true }, { "id": "bench-x3-rtx4060ti16-gemma9b", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 47, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "Gemma 2 9B Q4 fits.", "sourceUrl": "https://github.com/ollama/ollama/issues/6014", "testedAt": "2024-08-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000112, "perfPerWatt": 0.285, "tokensPerDollar": 0.094, "freshnessDays": 705, "isStale": true }, { "id": "bench-x3-rtx5060ti16-mistral7b", "deviceId": "rtx-5060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 74, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 180, "msrpUsd": 499, "sourceNote": "Blackwell entry-mid.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1iz3x4p/5060_ti_16gb_first/", "testedAt": "2025-04-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000071, "perfPerWatt": 0.411, "tokensPerDollar": 0.148, "freshnessDays": 446, "isStale": true }, { "id": "bench-x3-rtx5060ti16-gemma9b", "deviceId": "rtx-5060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 62, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 180, "msrpUsd": 499, "sourceNote": "Gemma 2 9B Q4 on 5060 Ti.", "sourceUrl": "https://github.com/ollama/ollama/issues/8421", "testedAt": "2025-04-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000085, "perfPerWatt": 0.344, "tokensPerDollar": 0.124, "freshnessDays": 440, "isStale": true }, { "id": "bench-x3-rtx3060-mistral7b", "deviceId": "rtx-3060-12gb", "deviceName": "NVIDIA GeForce RTX 3060 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 170, "msrpUsd": 329, "sourceNote": "Old reliable Ampere 12GB.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1ake7sd/3060_12gb_llm/", "testedAt": "2024-03-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000092, "perfPerWatt": 0.224, "tokensPerDollar": 0.116, "freshnessDays": 837, "isStale": true }, { "id": "bench-x3-rtx3060-gemma9b", "deviceId": "rtx-3060-12gb", "deviceName": "NVIDIA GeForce RTX 3060 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 170, "msrpUsd": 329, "sourceNote": "Gemma 2 9B Q4 tight on 12GB.", "sourceUrl": "https://github.com/ollama/ollama/issues/6014", "testedAt": "2024-06-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000109, "perfPerWatt": 0.188, "tokensPerDollar": 0.097, "freshnessDays": 766, "isStale": true }, { "id": "bench-x3-m3max-mistral7b", "deviceId": "apple-m3-max", "deviceName": "Apple M3 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 128, "tdpW": 60, "msrpUsd": 3999, "sourceNote": "M3 Max MLX 0.18.", "sourceUrl": "https://github.com/ml-explore/mlx/discussions/1244", "testedAt": "2024-04-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000813, "perfPerWatt": 0.867, "tokensPerDollar": 0.013, "freshnessDays": 829, "isStale": true }, { "id": "bench-x3-m3max-gemma9b", "deviceId": "apple-m3-max", "deviceName": "Apple M3 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 40, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 60, "msrpUsd": 3999, "sourceNote": "Gemma 2 9B MLX M3 Max.", "sourceUrl": "https://huggingface.co/mlx-community/gemma-2-9b-it-4bit", "testedAt": "2024-07-05", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001057, "perfPerWatt": 0.667, "tokensPerDollar": 0.01, "freshnessDays": 735, "isStale": true }, { "id": "bench-x3-m4pro-mistral7b", "deviceId": "apple-m4-pro", "deviceName": "Apple M4 Pro (20c GPU, 48GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 36, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "M4 Pro MLX.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-11-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000587, "perfPerWatt": 1.029, "tokensPerDollar": 0.018, "freshnessDays": 603, "isStale": true }, { "id": "bench-x3-m4pro-phi3", "deviceId": "apple-m4-pro", "deviceName": "Apple M4 Pro (20c GPU, 48GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "Phi-3 Mini MLX.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", "testedAt": "2024-12-01", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000271, "perfPerWatt": 2.229, "tokensPerDollar": 0.039, "freshnessDays": 586, "isStale": true }, { "id": "bench-x3-m4-mistral7b", "deviceId": "apple-m4", "deviceName": "Apple M4 (10c GPU, 24GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 22, "msrpUsd": 999, "sourceNote": "Mac mini M4.", "sourceUrl": "https://www.apple.com/mac-mini/specs/", "testedAt": "2024-11-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00048, "perfPerWatt": 1, "tokensPerDollar": 0.022, "freshnessDays": 595, "isStale": true }, { "id": "bench-x3-m4-phi3", "deviceId": "apple-m4", "deviceName": "Apple M4 (10c GPU, 24GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 22, "msrpUsd": 999, "sourceNote": "Phi-3 on Mac mini M4 base.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-12-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00022, "perfPerWatt": 2.182, "tokensPerDollar": 0.048, "freshnessDays": 572, "isStale": true }, { "id": "bench-x3-mmm4pro48-mistral7b", "deviceId": "mac-mini-m4-pro-48gb", "deviceName": "Apple Mac mini M4 Pro 48GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "Mac mini M4 Pro 48GB MLX.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-12-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000556, "perfPerWatt": 1.086, "tokensPerDollar": 0.019, "freshnessDays": 565, "isStale": true }, { "id": "bench-x3-mmm4pro48-gemma9b", "deviceId": "mac-mini-m4-pro-48gb", "deviceName": "Apple Mac mini M4 Pro 48GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 35, "msrpUsd": 1999, "sourceNote": "Gemma 2 9B Mac mini M4 Pro.", "sourceUrl": "https://huggingface.co/mlx-community/gemma-2-9b-it-4bit", "testedAt": "2025-01-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000755, "perfPerWatt": 0.8, "tokensPerDollar": 0.014, "freshnessDays": 548, "isStale": true }, { "id": "bench-x3-mmm4pro24-mistral7b", "deviceId": "mac-mini-m4-pro-24gb", "deviceName": "Apple Mac mini M4 Pro 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 35, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 35, "msrpUsd": 1399, "sourceNote": "Mac mini M4 Pro 24GB.", "sourceUrl": "https://www.apple.com/mac-mini/specs/", "testedAt": "2024-12-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000422, "perfPerWatt": 1, "tokensPerDollar": 0.025, "freshnessDays": 557, "isStale": true }, { "id": "bench-x3-mmm4pro24-phi3", "deviceId": "mac-mini-m4-pro-24gb", "deviceName": "Apple Mac mini M4 Pro 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 35, "msrpUsd": 1399, "sourceNote": "Phi-3 Mini MLX.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", "testedAt": "2025-01-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000205, "perfPerWatt": 2.057, "tokensPerDollar": 0.051, "freshnessDays": 541, "isStale": true }, { "id": "bench-x3-mmm4-16-phi3", "deviceId": "mac-mini-m4-16gb", "deviceName": "Apple Mac mini M4 16GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 45, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 22, "msrpUsd": 599, "sourceNote": "Phi-3 entry-level Mac mini.", "sourceUrl": "https://www.apple.com/mac-mini/specs/", "testedAt": "2024-11-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000141, "perfPerWatt": 2.045, "tokensPerDollar": 0.075, "freshnessDays": 587, "isStale": true }, { "id": "bench-x3-mmm4-16-mistral7b", "deviceId": "mac-mini-m4-16gb", "deviceName": "Apple Mac mini M4 16GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 20, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 22, "msrpUsd": 599, "sourceNote": "Tight VRAM 16GB.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-12-10", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000317, "perfPerWatt": 0.909, "tokensPerDollar": 0.033, "freshnessDays": 577, "isStale": true }, { "id": "bench-x3-mmm4-24-mistral7b", "deviceId": "mac-mini-m4-24gb", "deviceName": "Apple Mac mini M4 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 23, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 22, "msrpUsd": 799, "sourceNote": "Mac mini M4 24GB.", "sourceUrl": "https://www.apple.com/mac-mini/specs/", "testedAt": "2024-12-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000367, "perfPerWatt": 1.045, "tokensPerDollar": 0.029, "freshnessDays": 569, "isStale": true }, { "id": "bench-x3-mmm4-24-phi3", "deviceId": "mac-mini-m4-24gb", "deviceName": "Apple Mac mini M4 24GB", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 22, "msrpUsd": 799, "sourceNote": "Phi-3 on M4 24GB.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", "testedAt": "2025-01-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000162, "perfPerWatt": 2.364, "tokensPerDollar": 0.065, "freshnessDays": 552, "isStale": true }, { "id": "bench-x3-jetson-agx-mistral7b", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "mistral-7b-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "Jetson AGX llama.cpp CUDA.", "sourceUrl": "https://developer.nvidia.com/embedded/jetson-agx-orin-developer-kit", "testedAt": "2024-08-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00088, "perfPerWatt": 0.4, "tokensPerDollar": 0.012, "freshnessDays": 701, "isStale": true }, { "id": "bench-x3-jetson-agx-phi3", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 55, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "Phi-3 on Jetson AGX.", "sourceUrl": "https://github.com/NVIDIA-AI-IOT/jetson-containers", "testedAt": "2024-09-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000384, "perfPerWatt": 0.917, "tokensPerDollar": 0.028, "freshnessDays": 663, "isStale": true }, { "id": "bench-x3-orin-nano-super-phi3", "deviceId": "jetson-orin-nano-super", "deviceName": "NVIDIA Jetson Orin Nano Super 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 21, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 25, "msrpUsd": 249, "sourceNote": "Jetson Orin Nano Super 67 TOPS.", "sourceUrl": "https://developer.nvidia.com/embedded/learn/get-started-jetson-orin-nano-devkit", "testedAt": "2025-01-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000125, "perfPerWatt": 0.84, "tokensPerDollar": 0.084, "freshnessDays": 534, "isStale": true }, { "id": "bench-x3-orin-nano-super-mistral", "deviceId": "jetson-orin-nano-super", "deviceName": "NVIDIA Jetson Orin Nano Super 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "mistral-7b-q4", "tokensPerSec": 8, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 25, "msrpUsd": 249, "sourceNote": "Mistral 7B Q4 tight fit.", "sourceUrl": "https://github.com/NVIDIA-AI-IOT/jetson-containers", "testedAt": "2025-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000329, "perfPerWatt": 0.32, "tokensPerDollar": 0.032, "freshnessDays": 521, "isStale": true }, { "id": "bench-x3-pi-aihat-phi3", "deviceId": "raspberry-pi-ai-hat-plus", "deviceName": "Raspberry Pi AI Hat+ (Hailo-8L 13 TOPS)", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 6, "quantization": "INT8", "contextLength": 4096, "vramGB": 0, "tdpW": 8, "msrpUsd": 70, "sourceNote": "Hailo-8L NPU on Pi 5.", "sourceUrl": "https://www.raspberrypi.com/products/ai-hat/", "testedAt": "2024-12-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000123, "perfPerWatt": 0.75, "tokensPerDollar": 0.086, "freshnessDays": 575, "isStale": true }, { "id": "bench-x3-pi-aihat-whisper", "deviceId": "raspberry-pi-ai-hat-plus", "deviceName": "Raspberry Pi AI Hat+ (Hailo-8L 13 TOPS)", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "whisper-large-v3", "audioRtfx": 1.2, "quantization": "INT8", "contextLength": 0, "vramGB": 0, "tdpW": 8, "msrpUsd": 70, "sourceNote": "Hailo Whisper tiny optimized path.", "sourceUrl": "https://hailo.ai/products/ai-accelerators/", "testedAt": "2025-01-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 538, "isStale": true }, { "id": "bench-x3-xeon6-mistral7b", "deviceId": "intel-xeon-6-6980p", "deviceName": "Intel Xeon 6980P (128 cores)", "brand": "Intel", "deviceClass": "cpu-only", "workload": "mistral-7b-q4", "tokensPerSec": 18, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 500, "msrpUsd": 17800, "sourceNote": "CPU-only llama.cpp AVX-512.", "sourceUrl": "https://www.intel.com/content/www/us/en/products/sku/240777/intel-xeon-6980p-processor-504m-cache-2-00-ghz/specifications.html", "testedAt": "2024-11-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.010452, "perfPerWatt": 0.036, "tokensPerDollar": 0.001, "freshnessDays": 591, "isStale": true }, { "id": "bench-x3-xeon6-phi3", "deviceId": "intel-xeon-6-6980p", "deviceName": "Intel Xeon 6980P (128 cores)", "brand": "Intel", "deviceClass": "cpu-only", "workload": "phi3-mini-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 500, "msrpUsd": 17800, "sourceNote": "Phi-3 Mini CPU on Xeon 6.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9521", "testedAt": "2024-12-06", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00448, "perfPerWatt": 0.084, "tokensPerDollar": 0.002, "freshnessDays": 581, "isStale": true }, { "id": "bench-x3-tr7995wx-mistral7b", "deviceId": "amd-threadripper-7995wx", "deviceName": "AMD Threadripper Pro 7995WX (96 cores)", "brand": "AMD", "deviceClass": "cpu-only", "workload": "mistral-7b-q4", "tokensPerSec": 14, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 350, "msrpUsd": 9999, "sourceNote": "CPU-only Zen 4.", "sourceUrl": "https://www.amd.com/en/products/processors/workstation/ryzen-threadripper.html", "testedAt": "2024-09-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.007549, "perfPerWatt": 0.04, "tokensPerDollar": 0.001, "freshnessDays": 650, "isStale": true }, { "id": "bench-x3-tr7995wx-phi3", "deviceId": "amd-threadripper-7995wx", "deviceName": "AMD Threadripper Pro 7995WX (96 cores)", "brand": "AMD", "deviceClass": "cpu-only", "workload": "phi3-mini-q4", "tokensPerSec": 36, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 350, "msrpUsd": 9999, "sourceNote": "Phi-3 Mini CPU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/8201", "testedAt": "2024-10-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002936, "perfPerWatt": 0.103, "tokensPerDollar": 0.004, "freshnessDays": 634, "isStale": true }, { "id": "bench-x3-m3pro-mistral7b", "deviceId": "apple-m3-pro", "deviceName": "Apple M3 Pro (18c GPU, 36GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 36, "tdpW": 30, "msrpUsd": 2499, "sourceNote": "M3 Pro MLX.", "sourceUrl": "https://github.com/ml-explore/mlx/discussions/1244", "testedAt": "2024-05-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000943, "perfPerWatt": 0.933, "tokensPerDollar": 0.011, "freshnessDays": 793, "isStale": true }, { "id": "bench-x3-m3pro-phi3", "deviceId": "apple-m3-pro", "deviceName": "Apple M3 Pro (18c GPU, 36GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 62, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 36, "tdpW": 30, "msrpUsd": 2499, "sourceNote": "Phi-3 Mini M3 Pro.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", "testedAt": "2024-07-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000426, "perfPerWatt": 2.067, "tokensPerDollar": 0.025, "freshnessDays": 714, "isStale": true }, { "id": "bench-x3-m2max-mistral7b", "deviceId": "apple-m2-max", "deviceName": "Apple M2 Max (38c GPU, 96GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 42, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 96, "tdpW": 50, "msrpUsd": 3499, "sourceNote": "M2 Max MLX 0.16.", "sourceUrl": "https://github.com/ml-explore/mlx/discussions/1244", "testedAt": "2024-04-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000881, "perfPerWatt": 0.84, "tokensPerDollar": 0.012, "freshnessDays": 811, "isStale": true }, { "id": "bench-x3-m2max-gemma9b", "deviceId": "apple-m2-max", "deviceName": "Apple M2 Max (38c GPU, 96GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "gemma-2-9b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 50, "msrpUsd": 3499, "sourceNote": "Gemma 2 9B MLX.", "sourceUrl": "https://huggingface.co/mlx-community/gemma-2-9b-it-4bit", "testedAt": "2024-07-23", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001156, "perfPerWatt": 0.64, "tokensPerDollar": 0.009, "freshnessDays": 717, "isStale": true }, { "id": "bench-x3-m2pro-mistral7b", "deviceId": "apple-m2-pro", "deviceName": "Apple M2 Pro (19c GPU, 32GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "mistral-7b-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 28, "msrpUsd": 1999, "sourceNote": "M2 Pro MLX.", "sourceUrl": "https://github.com/ml-explore/mlx/discussions/1244", "testedAt": "2024-03-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00088, "perfPerWatt": 0.857, "tokensPerDollar": 0.012, "freshnessDays": 832, "isStale": true }, { "id": "bench-x3-m2pro-phi3", "deviceId": "apple-m2-pro", "deviceName": "Apple M2 Pro (19c GPU, 32GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "phi3-mini-q4", "tokensPerSec": 54, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 32, "tdpW": 28, "msrpUsd": 1999, "sourceNote": "Phi-3 Mini M2 Pro.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", "testedAt": "2024-05-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000391, "perfPerWatt": 1.929, "tokensPerDollar": 0.027, "freshnessDays": 775, "isStale": true }, { "id": "bench-x3-cu9-285k-mistral7b", "deviceId": "intel-core-ultra-9-285k", "deviceName": "Intel Core Ultra 9 285K", "brand": "Intel", "deviceClass": "cpu-only", "workload": "mistral-7b-q4", "tokensPerSec": 9, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 250, "msrpUsd": 589, "sourceNote": "Arrow Lake CPU AVX2.", "sourceUrl": "https://www.intel.com/content/www/us/en/products/sku/241060/intel-core-ultra-9-processor-285k/specifications.html", "testedAt": "2024-10-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000692, "perfPerWatt": 0.036, "tokensPerDollar": 0.015, "freshnessDays": 623, "isStale": true }, { "id": "bench-x3-cu9-285k-phi3", "deviceId": "intel-core-ultra-9-285k", "deviceName": "Intel Core Ultra 9 285K", "brand": "Intel", "deviceClass": "cpu-only", "workload": "phi3-mini-q4", "tokensPerSec": 24, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 250, "msrpUsd": 589, "sourceNote": "Phi-3 CPU.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9000", "testedAt": "2024-11-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000259, "perfPerWatt": 0.096, "tokensPerDollar": 0.041, "freshnessDays": 608, "isStale": true }, { "id": "bench-x3-cu258v-phi3", "deviceId": "intel-core-ultra-258v", "deviceName": "Intel Core Ultra 258V (Lunar Lake NPU 48 TOPS)", "brand": "Intel", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 17, "msrpUsd": 1299, "sourceNote": "Lunar Lake NPU via OpenVINO.", "sourceUrl": "https://www.intel.com/content/www/us/en/products/sku/240955/intel-core-ultra-7-processor-258v/specifications.html", "testedAt": "2024-09-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00049, "perfPerWatt": 1.647, "tokensPerDollar": 0.022, "freshnessDays": 656, "isStale": true }, { "id": "bench-x3-cu258v-mistral", "deviceId": "intel-core-ultra-258v", "deviceName": "Intel Core Ultra 258V (Lunar Lake NPU 48 TOPS)", "brand": "Intel", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 13, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 17, "msrpUsd": 1299, "sourceNote": "Mistral 7B OpenVINO INT4.", "sourceUrl": "https://www.intel.com/content/www/us/en/developer/articles/technical/openvino-llms-genai.html", "testedAt": "2024-10-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001056, "perfPerWatt": 0.765, "tokensPerDollar": 0.01, "freshnessDays": 640, "isStale": true }, { "id": "bench-x3-cu288v-phi3", "deviceId": "intel-core-ultra-9-288v", "deviceName": "Intel Core Ultra 9 288V (Lunar Lake)", "brand": "Intel", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 30, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 17, "msrpUsd": 1399, "sourceNote": "Top Lunar Lake SKU.", "sourceUrl": "https://www.intel.com/content/www/us/en/products/sku/240952/intel-core-ultra-9-processor-288v/specifications.html", "testedAt": "2024-10-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000493, "perfPerWatt": 1.765, "tokensPerDollar": 0.021, "freshnessDays": 618, "isStale": true }, { "id": "bench-x3-ai9hx370-phi3", "deviceId": "amd-ryzen-ai-9-hx-370", "deviceName": "AMD Ryzen AI 9 HX 370 (Strix Point NPU 50 TOPS)", "brand": "AMD", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 28, "msrpUsd": 1499, "sourceNote": "Strix Point XDNA 2 NPU.", "sourceUrl": "https://www.amd.com/en/products/processors/laptop/ryzen.html", "testedAt": "2024-09-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000495, "perfPerWatt": 1.143, "tokensPerDollar": 0.021, "freshnessDays": 674, "isStale": true }, { "id": "bench-x3-ai9hx370-mistral", "deviceId": "amd-ryzen-ai-9-hx-370", "deviceName": "AMD Ryzen AI 9 HX 370 (Strix Point NPU 50 TOPS)", "brand": "AMD", "deviceClass": "npu", "workload": "mistral-7b-q4", "tokensPerSec": 16, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 28, "msrpUsd": 1499, "sourceNote": "Mistral 7B Ryzen AI Software 1.3.", "sourceUrl": "https://ryzenai.docs.amd.com/", "testedAt": "2024-10-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00099, "perfPerWatt": 0.571, "tokensPerDollar": 0.011, "freshnessDays": 632, "isStale": true }, { "id": "bench-x3-ai9-365-phi3", "deviceId": "amd-ryzen-ai-9-365", "deviceName": "AMD Ryzen AI 9 365 (Strix Point)", "brand": "AMD", "deviceClass": "npu", "workload": "phi3-mini-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 28, "msrpUsd": 1299, "sourceNote": "Ryzen AI 9 365 XDNA 2.", "sourceUrl": "https://www.amd.com/en/products/processors/laptop/ryzen.html", "testedAt": "2024-09-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00049, "perfPerWatt": 1, "tokensPerDollar": 0.022, "freshnessDays": 666, "isStale": true }, { "id": "bench-x3-rx7800xt-mistral7b", "deviceId": "rx-7800-xt", "deviceName": "AMD Radeon RX 7800 XT 16GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 263, "msrpUsd": 499, "sourceNote": "RDNA 3 16GB ROCm 6.2.", "sourceUrl": "https://github.com/ROCm/ROCm/discussions/2986", "testedAt": "2024-07-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000091, "perfPerWatt": 0.221, "tokensPerDollar": 0.116, "freshnessDays": 721, "isStale": true }, { "id": "bench-x3-rx7800xt-phi3", "deviceId": "rx-7800-xt", "deviceName": "AMD Radeon RX 7800 XT 16GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 132, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 263, "msrpUsd": 499, "sourceNote": "Phi-3 Mini on RDNA 3.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-08-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00004, "perfPerWatt": 0.502, "tokensPerDollar": 0.265, "freshnessDays": 679, "isStale": true }, { "id": "bench-x3-rx7700xt-mistral7b", "deviceId": "rx-7700-xt", "deviceName": "AMD Radeon RX 7700 XT 12GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 12, "tdpW": 245, "msrpUsd": 449, "sourceNote": "RDNA 3 12GB.", "sourceUrl": "https://www.amd.com/en/products/graphics/amd-radeon-rx-7700-xt.html", "testedAt": "2024-07-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000099, "perfPerWatt": 0.196, "tokensPerDollar": 0.107, "freshnessDays": 714, "isStale": true }, { "id": "bench-x3-rx7600-phi3", "deviceId": "rx-7600", "deviceName": "AMD Radeon RX 7600 8GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 88, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 165, "msrpUsd": 269, "sourceNote": "Phi-3 Mini RDNA 3 entry.", "sourceUrl": "https://www.amd.com/en/products/graphics/amd-radeon-rx-7600.html", "testedAt": "2024-08-06", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000032, "perfPerWatt": 0.533, "tokensPerDollar": 0.327, "freshnessDays": 703, "isStale": true }, { "id": "bench-x3-a770-mistral", "deviceId": "arc-a770", "deviceName": "Intel Arc A770 16GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 32, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 225, "msrpUsd": 349, "sourceNote": "Arc Alchemist IPEX-LLM.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2024-05-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000115, "perfPerWatt": 0.142, "tokensPerDollar": 0.092, "freshnessDays": 779, "isStale": true }, { "id": "bench-x3-a770-phi3", "deviceId": "arc-a770", "deviceName": "Intel Arc A770 16GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 72, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 225, "msrpUsd": 349, "sourceNote": "Phi-3 Mini IPEX-LLM.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm/issues/10220", "testedAt": "2024-06-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000051, "perfPerWatt": 0.32, "tokensPerDollar": 0.206, "freshnessDays": 761, "isStale": true }, { "id": "bench-x3-mi250x-mistral7b", "deviceId": "mi250x", "deviceName": "AMD Instinct MI250X 128GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 135, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 128, "tdpW": 560, "msrpUsd": 12000, "sourceNote": "CDNA 2 ROCm 6.0.", "sourceUrl": "https://www.amd.com/en/products/accelerators/instinct/mi200/mi250x.html", "testedAt": "2024-04-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00094, "perfPerWatt": 0.241, "tokensPerDollar": 0.011, "freshnessDays": 823, "isStale": true }, { "id": "bench-x3-mi250x-gemma9b", "deviceId": "mi250x", "deviceName": "AMD Instinct MI250X 128GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 110, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 560, "msrpUsd": 12000, "sourceNote": "Gemma 2 9B on MI250X.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/4756", "testedAt": "2024-07-31", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001153, "perfPerWatt": 0.196, "tokensPerDollar": 0.009, "freshnessDays": 709, "isStale": true }, { "id": "bench-x3-mi210-mistral7b", "deviceId": "mi210", "deviceName": "AMD Instinct MI210 64GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 105, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 300, "msrpUsd": 7500, "sourceNote": "CDNA 2 mid-tier.", "sourceUrl": "https://www.amd.com/en/products/accelerators/instinct/mi200/mi210.html", "testedAt": "2024-04-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000755, "perfPerWatt": 0.35, "tokensPerDollar": 0.014, "freshnessDays": 817, "isStale": true }, { "id": "bench-x3-mi210-phi3", "deviceId": "mi210", "deviceName": "AMD Instinct MI210 64GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 175, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 64, "tdpW": 300, "msrpUsd": 7500, "sourceNote": "Phi-3 Mini MI210.", "sourceUrl": "https://github.com/ROCm/ROCm/discussions/2986", "testedAt": "2024-05-19", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000453, "perfPerWatt": 0.583, "tokensPerDollar": 0.023, "freshnessDays": 782, "isStale": true }, { "id": "bench-x3-tpu-v5e-mistral", "deviceId": "tpu-v5e", "deviceName": "Google TPU v5e", "brand": "Google", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 195, "quantization": "INT8", "contextLength": 4096, "vramGB": 16, "tdpW": 170, "msrpUsd": 0, "sourceNote": "Cloud TPU v5e JAX/MaxText.", "sourceUrl": "https://cloud.google.com/tpu/docs/v5e", "testedAt": "2024-06-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0, "perfPerWatt": 1.147, "freshnessDays": 755, "isStale": true }, { "id": "bench-x3-tpu-v5e-phi3", "deviceId": "tpu-v5e", "deviceName": "Google TPU v5e", "brand": "Google", "deviceClass": "asic", "workload": "phi3-mini-q4", "tokensPerSec": 320, "quantization": "INT8", "contextLength": 4096, "vramGB": 16, "tdpW": 170, "msrpUsd": 0, "sourceNote": "Phi-3 Mini on TPU v5e.", "sourceUrl": "https://cloud.google.com/tpu/docs/v5e", "testedAt": "2024-08-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0, "perfPerWatt": 1.882, "freshnessDays": 683, "isStale": true }, { "id": "bench-x3-tpu-v5p-mistral", "deviceId": "tpu-v5p-trillium", "deviceName": "Google TPU v5p (Trillium)", "brand": "Google", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 410, "quantization": "INT8", "contextLength": 4096, "vramGB": 95, "tdpW": 300, "msrpUsd": 0, "sourceNote": "Trillium 4.7x perf vs v5e.", "sourceUrl": "https://cloud.google.com/blog/products/compute/introducing-trillium-6th-gen-tpus", "testedAt": "2024-12-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0, "perfPerWatt": 1.367, "freshnessDays": 576, "isStale": true }, { "id": "bench-x3-tpu-v5p-qwen14b", "deviceId": "tpu-v5p-trillium", "deviceName": "Google TPU v5p (Trillium)", "brand": "Google", "deviceClass": "asic", "workload": "qwen2.5-14b-q4", "tokensPerSec": 195, "quantization": "INT8", "contextLength": 8192, "vramGB": 95, "tdpW": 300, "msrpUsd": 0, "sourceNote": "Qwen 2.5 14B Trillium.", "sourceUrl": "https://cloud.google.com/blog/products/ai-machine-learning/google-cloud-tpu-trillium-pricing-and-availability", "testedAt": "2025-02-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0, "perfPerWatt": 0.65, "freshnessDays": 505, "isStale": true }, { "id": "bench-x3-trn2-mistral", "deviceId": "aws-trainium2", "deviceName": "AWS Trainium2 Trn2", "brand": "AWS", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 280, "quantization": "INT8", "contextLength": 4096, "vramGB": 96, "tdpW": 350, "msrpUsd": 0, "sourceNote": "Trainium2 NeuronX SDK 2.20.", "sourceUrl": "https://aws.amazon.com/ai/machine-learning/trainium/", "testedAt": "2024-12-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0, "perfPerWatt": 0.8, "freshnessDays": 583, "isStale": true }, { "id": "bench-x3-trn2-qwen14b", "deviceId": "aws-trainium2", "deviceName": "AWS Trainium2 Trn2", "brand": "AWS", "deviceClass": "asic", "workload": "qwen2.5-14b-q4", "tokensPerSec": 158, "quantization": "INT8", "contextLength": 8192, "vramGB": 96, "tdpW": 350, "msrpUsd": 0, "sourceNote": "Qwen 2.5 14B Trainium2.", "sourceUrl": "https://aws.amazon.com/ai/machine-learning/trainium/", "testedAt": "2025-01-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0, "perfPerWatt": 0.451, "freshnessDays": 536, "isStale": true }, { "id": "bench-x3-cerebras-mistral", "deviceId": "cerebras-wse-3", "deviceName": "Cerebras WSE-3", "brand": "Cerebras", "deviceClass": "asic", "workload": "mistral-7b-q4", "tokensPerSec": 1850, "quantization": "FP16", "contextLength": 4096, "vramGB": 44000, "tdpW": 23000, "msrpUsd": 2000000, "sourceNote": "Wafer-scale Cerebras Inference API.", "sourceUrl": "https://www.cerebras.ai/inference", "testedAt": "2024-08-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.011427, "perfPerWatt": 0.08, "tokensPerDollar": 0.001, "freshnessDays": 681, "isStale": true }, { "id": "bench-x3-cerebras-gemma9b", "deviceId": "cerebras-wse-3", "deviceName": "Cerebras WSE-3", "brand": "Cerebras", "deviceClass": "asic", "workload": "gemma-2-9b-q4", "tokensPerSec": 1420, "quantization": "FP16", "contextLength": 8192, "vramGB": 44000, "tdpW": 23000, "msrpUsd": 2000000, "sourceNote": "Gemma 2 9B on WSE-3.", "sourceUrl": "https://www.cerebras.ai/inference", "testedAt": "2024-10-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.014887, "perfPerWatt": 0.062, "tokensPerDollar": 0.001, "freshnessDays": 618, "isStale": true }, { "id": "bench-x3-gh200-mistral7b", "deviceId": "gh200", "deviceName": "NVIDIA GH200 480GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 305, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 144, "tdpW": 1000, "msrpUsd": 45000, "sourceNote": "Grace-Hopper superchip.", "sourceUrl": "https://www.nvidia.com/en-us/data-center/grace-hopper-superchip/", "testedAt": "2024-08-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001559, "perfPerWatt": 0.305, "tokensPerDollar": 0.007, "freshnessDays": 705, "isStale": true }, { "id": "bench-x3-gh200-gemma9b", "deviceId": "gh200", "deviceName": "NVIDIA GH200 480GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 248, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 144, "tdpW": 1000, "msrpUsd": 45000, "sourceNote": "Gemma 2 9B GH200.", "sourceUrl": "https://www.nvidia.com/en-us/data-center/grace-hopper-superchip/", "testedAt": "2024-10-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001918, "perfPerWatt": 0.248, "tokensPerDollar": 0.006, "freshnessDays": 622, "isStale": true }, { "id": "bench-x3-rtx5060-phi3", "deviceId": "rtx-5060", "deviceName": "NVIDIA GeForce RTX 5060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 145, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 145, "msrpUsd": 299, "sourceNote": "Blackwell entry.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1j2c8df/5060_phi3/", "testedAt": "2025-04-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000022, "perfPerWatt": 1, "tokensPerDollar": 0.485, "freshnessDays": 441, "isStale": true }, { "id": "bench-x3-rtx5060-mistral", "deviceId": "rtx-5060", "deviceName": "NVIDIA GeForce RTX 5060 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 64, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 145, "msrpUsd": 299, "sourceNote": "Tight VRAM Mistral 7B.", "sourceUrl": "https://github.com/ollama/ollama/issues/8421", "testedAt": "2025-05-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000049, "perfPerWatt": 0.441, "tokensPerDollar": 0.214, "freshnessDays": 434, "isStale": true }, { "id": "bench-x3-rtx5060ti8-phi3", "deviceId": "rtx-5060-ti-8gb", "deviceName": "NVIDIA GeForce RTX 5060 Ti 8GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 158, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 180, "msrpUsd": 379, "sourceNote": "Blackwell 8GB Ti.", "sourceUrl": "https://github.com/ollama/ollama/issues/8421", "testedAt": "2025-04-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000025, "perfPerWatt": 0.878, "tokensPerDollar": 0.417, "freshnessDays": 444, "isStale": true }, { "id": "bench-x3-a40-mistral", "deviceId": "a40", "deviceName": "NVIDIA A40 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 88, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 300, "msrpUsd": 5500, "sourceNote": "Ampere A40.", "sourceUrl": "https://www.nvidia.com/en-us/data-center/a40/", "testedAt": "2024-04-11", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000661, "perfPerWatt": 0.293, "tokensPerDollar": 0.016, "freshnessDays": 820, "isStale": true }, { "id": "bench-x3-a40-phi3", "deviceId": "a40", "deviceName": "NVIDIA A40 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 162, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 48, "tdpW": 300, "msrpUsd": 5500, "sourceNote": "Phi-3 Mini A40.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-05-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000359, "perfPerWatt": 0.54, "tokensPerDollar": 0.029, "freshnessDays": 780, "isStale": true }, { "id": "bench-x3-a5000-mistral", "deviceId": "rtx-a5000", "deviceName": "NVIDIA RTX A5000 24GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 64, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 230, "msrpUsd": 2500, "sourceNote": "Ampere workstation A5000.", "sourceUrl": "https://www.nvidia.com/en-us/design-visualization/rtx-a5000/", "testedAt": "2024-04-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000413, "perfPerWatt": 0.278, "tokensPerDollar": 0.026, "freshnessDays": 805, "isStale": true }, { "id": "bench-x3-a5000-phi3", "deviceId": "rtx-a5000", "deviceName": "NVIDIA RTX A5000 24GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 122, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 24, "tdpW": 230, "msrpUsd": 2500, "sourceNote": "Phi-3 Mini A5000.", "sourceUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", "testedAt": "2024-06-13", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000217, "perfPerWatt": 0.53, "tokensPerDollar": 0.049, "freshnessDays": 757, "isStale": true }, { "id": "bench-x3-a4000-phi3", "deviceId": "rtx-a4000", "deviceName": "NVIDIA RTX A4000 16GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 140, "msrpUsd": 1200, "sourceNote": "Workstation A4000.", "sourceUrl": "https://www.nvidia.com/en-us/design-visualization/rtx-a4000/", "testedAt": "2024-05-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000138, "perfPerWatt": 0.657, "tokensPerDollar": 0.077, "freshnessDays": 799, "isStale": true }, { "id": "bench-x3-a4000-mistral", "deviceId": "rtx-a4000", "deviceName": "NVIDIA RTX A4000 16GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "mistral-7b-q4", "tokensPerSec": 48, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 140, "msrpUsd": 1200, "sourceNote": "Mistral 7B A4000.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/8201", "testedAt": "2024-06-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000264, "perfPerWatt": 0.343, "tokensPerDollar": 0.04, "freshnessDays": 754, "isStale": true }, { "id": "bench-x3-orin-nano-mistral", "deviceId": "jetson-orin-nano-8gb", "deviceName": "NVIDIA Jetson Orin Nano 8GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "mistral-7b-q4", "tokensPerSec": 5.5, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 15, "msrpUsd": 499, "sourceNote": "Original Orin Nano (pre-Super).", "sourceUrl": "https://developer.nvidia.com/embedded/jetson-orin-nano-developer-kit", "testedAt": "2024-05-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000959, "perfPerWatt": 0.367, "tokensPerDollar": 0.011, "freshnessDays": 771, "isStale": true }, { "id": "bench-x3-orin-nx-mistral", "deviceId": "jetson-orin-nx-16gb", "deviceName": "NVIDIA Jetson Orin NX 16GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "mistral-7b-q4", "tokensPerSec": 10, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 25, "msrpUsd": 699, "sourceNote": "Edge Orin NX.", "sourceUrl": "https://developer.nvidia.com/embedded/jetson-orin-nx-16gb-developer-kit", "testedAt": "2024-06-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000739, "perfPerWatt": 0.4, "tokensPerDollar": 0.014, "freshnessDays": 750, "isStale": true }, { "id": "bench-x3-orin-nx-phi3", "deviceId": "jetson-orin-nx-16gb", "deviceName": "NVIDIA Jetson Orin NX 16GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 22, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 16, "tdpW": 25, "msrpUsd": 699, "sourceNote": "Phi-3 on Orin NX.", "sourceUrl": "https://github.com/NVIDIA-AI-IOT/jetson-containers", "testedAt": "2024-07-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000336, "perfPerWatt": 0.88, "tokensPerDollar": 0.031, "freshnessDays": 732, "isStale": true }, { "id": "bench-x3-pi5-phi3", "deviceId": "raspberry-pi-5", "deviceName": "Raspberry Pi 5 (8GB)", "brand": "Raspberry Pi", "deviceClass": "edge", "workload": "phi3-mini-q4", "tokensPerSec": 3.5, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 0, "tdpW": 12, "msrpUsd": 80, "sourceNote": "ARM CPU-only Pi 5 8GB.", "sourceUrl": "https://www.raspberrypi.com/products/raspberry-pi-5/", "testedAt": "2024-04-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000242, "perfPerWatt": 0.292, "tokensPerDollar": 0.044, "freshnessDays": 813, "isStale": true }, { "id": "bench-x3-rtx4090-2x-qwen14b", "deviceId": "rtx-4090-2x", "deviceName": "2× NVIDIA RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 92, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 900, "msrpUsd": 3198, "sourceNote": "2x4090 NVLink-less tensor parallel.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1ds2xnh/2x_4090_qwen14b/", "testedAt": "2024-09-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000367, "perfPerWatt": 0.102, "tokensPerDollar": 0.029, "freshnessDays": 657, "isStale": true }, { "id": "bench-x3-rtx4090-2x-gemma9b", "deviceId": "rtx-4090-2x", "deviceName": "2× NVIDIA RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 900, "msrpUsd": 3198, "sourceNote": "Dual 4090 Gemma 2.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/4567", "testedAt": "2024-10-29", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000245, "perfPerWatt": 0.153, "tokensPerDollar": 0.043, "freshnessDays": 619, "isStale": true }, { "id": "bench-x3-rtx3090-4x-qwen14b", "deviceId": "rtx-3090-4x", "deviceName": "4× NVIDIA RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 96, "tdpW": 1400, "msrpUsd": 4500, "sourceNote": "Quad 3090 community rig.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1eq2c4f/4x_3090_build_qwen/", "testedAt": "2024-08-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00061, "perfPerWatt": 0.056, "tokensPerDollar": 0.017, "freshnessDays": 684, "isStale": true }, { "id": "bench-x3-h100-8x-qwen14b", "deviceId": "h100-sxm5-8x", "deviceName": "8× NVIDIA H100 SXM5 80GB (DGX H100)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 1080, "quantization": "Q4_K_M", "contextLength": 8192, "batchSize": 8, "vramGB": 640, "tdpW": 5600, "msrpUsd": 200000, "sourceNote": "DGX H100 batch 8 aggregate.", "sourceUrl": "https://www.nvidia.com/en-us/data-center/dgx-h100/", "testedAt": "2024-09-12", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001957, "perfPerWatt": 0.193, "tokensPerDollar": 0.005, "freshnessDays": 666, "isStale": true }, { "id": "bench-x3-h100-8x-gemma9b", "deviceId": "h100-sxm5-8x", "deviceName": "8× NVIDIA H100 SXM5 80GB (DGX H100)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "gemma-2-9b-q4", "tokensPerSec": 1450, "quantization": "Q4_K_M", "contextLength": 8192, "batchSize": 8, "vramGB": 640, "tdpW": 5600, "msrpUsd": 200000, "sourceNote": "DGX H100 Gemma 2 9B batch 8.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2024-10-04", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001458, "perfPerWatt": 0.259, "tokensPerDollar": 0.007, "freshnessDays": 644, "isStale": true }, { "id": "bench-x3-a750-phi3", "deviceId": "arc-a750", "deviceName": "Intel Arc A750 8GB", "brand": "Intel", "deviceClass": "consumer-gpu", "workload": "phi3-mini-q4", "tokensPerSec": 58, "quantization": "Q4_K_M", "contextLength": 4096, "vramGB": 8, "tdpW": 225, "msrpUsd": 249, "sourceNote": "Arc Alchemist 8GB IPEX-LLM.", "sourceUrl": "https://github.com/intel-analytics/ipex-llm", "testedAt": "2024-05-15", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000045, "perfPerWatt": 0.258, "tokensPerDollar": 0.233, "freshnessDays": 786, "isStale": true }, { "id": "bench-x3-m3ultra-l3-8b-128k", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "llama3-8b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 131072, "vramGB": 512, "tdpW": 100, "msrpUsd": 9999, "sourceNote": "128k context — unified mem absorbs KV cache.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2025-03-28", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002781, "perfPerWatt": 0.38, "tokensPerDollar": 0.004, "freshnessDays": 469, "isStale": true }, { "id": "bench-x3-mi300x-l3-8b-128k", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 122, "quantization": "Q4_K_M", "contextLength": 131072, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "128k ctx MI300X vLLM.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/4756", "testedAt": "2025-04-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.0013, "perfPerWatt": 0.163, "tokensPerDollar": 0.008, "freshnessDays": 458, "isStale": true }, { "id": "bench-x3-h200-l3-8b-128k", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 215, "quantization": "Q4_K_M", "contextLength": 131072, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "128k context H200.", "sourceUrl": "https://mlcommons.org/en/inference-datacenter-4-1/", "testedAt": "2025-02-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001475, "perfPerWatt": 0.307, "tokensPerDollar": 0.007, "freshnessDays": 521, "isStale": true }, { "id": "bench-x3-b200-l3-8b-128k", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 360, "quantization": "Q4_K_M", "contextLength": 131072, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "128k Blackwell DC.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2025-03-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001174, "perfPerWatt": 0.36, "tokensPerDollar": 0.009, "freshnessDays": 477, "isStale": true }, { "id": "bench-x3-rtx5090-l3-70b-batch8", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 78, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Batch 8 70B with offload.", "sourceUrl": "https://github.com/vllm-project/vllm/pull/6789", "testedAt": "2025-03-25", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000271, "perfPerWatt": 0.136, "tokensPerDollar": 0.039, "freshnessDays": 472, "isStale": true }, { "id": "bench-x3-mi300x-l3-8b-batch8", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "llama3-8b-q4", "tokensPerSec": 920, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "Batch 8 vLLM ROCm.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/4756", "testedAt": "2024-12-08", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000172, "perfPerWatt": 1.227, "tokensPerDollar": 0.061, "freshnessDays": 579, "isStale": true }, { "id": "bench-x3-h200-l3-70b-batch8", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "llama3-70b-q4", "tokensPerSec": 540, "quantization": "Q4_K_M", "contextLength": 4096, "batchSize": 8, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Batch 8 70B TRT-LLM.", "sourceUrl": "https://developer.nvidia.com/blog/nvidia-tensorrt-llm-supercharges-large-language-model-inference-on-nvidia-h100-gpus/", "testedAt": "2024-11-22", "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000587, "perfPerWatt": 0.771, "tokensPerDollar": 0.018, "freshnessDays": 595, "isStale": true }, { "id": "bench-x3-rtx5090-sdxl", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 18, "quantization": "FP16", "contextLength": 0, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "SDXL base 30-step on Blackwell.", "sourceUrl": "https://www.reddit.com/r/StableDiffusion/comments/1iq5xd9/rtx_5090_sdxl_speed/", "testedAt": "2025-02-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 517, "isStale": true }, { "id": "bench-x3-m4max-sdxl", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "sdxl-1024", "imagesPerMin": 6, "quantization": "FP16", "contextLength": 0, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "SDXL Diffusers MLX path.", "sourceUrl": "https://github.com/argmaxinc/DiffusionKit", "testedAt": "2025-01-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 526, "isStale": true }, { "id": "bench-x3-rtx3090-sdxl", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 7.5, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "SDXL ComfyUI 3090.", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2024-05-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 775, "isStale": true }, { "id": "bench-x3-rx7900xtx-sdxl", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "sdxl-1024", "imagesPerMin": 6.5, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "SDXL ROCm 6.2 Diffusers.", "sourceUrl": "https://github.com/comfyanonymous/ComfyUI/issues/2453", "testedAt": "2024-09-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 670, "isStale": true }, { "id": "bench-x3-mi300x-sdxl", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "sdxl-1024", "imagesPerMin": 22, "quantization": "FP16", "contextLength": 0, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "SDXL on MI300X ROCm.", "sourceUrl": "https://www.amd.com/en/products/accelerators/instinct/mi300/mi300x.html", "testedAt": "2024-12-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 565, "isStale": true }, { "id": "bench-x3-rtx4080s-whisper", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 48, "quantization": "FP16", "contextLength": 0, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "faster-whisper 4080S.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/issues/911", "testedAt": "2024-10-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 636, "isStale": true }, { "id": "bench-x3-rtx3090-whisper", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 32, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Whisper 3090.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/discussions/1832", "testedAt": "2024-06-25", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 745, "isStale": true }, { "id": "bench-x3-rx7900xtx-whisper", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "whisper-large-v3", "audioRtfx": 22, "quantization": "FP16", "contextLength": 0, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "whisper.cpp HIP RDNA 3.", "sourceUrl": "https://github.com/ggerganov/whisper.cpp/pull/2113", "testedAt": "2024-10-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 630, "isStale": true }, { "id": "bench-x3-mi300x-whisper", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "whisper-large-v3", "audioRtfx": 95, "quantization": "FP16", "contextLength": 0, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "Whisper MI300X ROCm.", "sourceUrl": "https://github.com/SYSTRAN/faster-whisper/issues/911", "testedAt": "2025-01-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 544, "isStale": true }, { "id": "bench-x3-jetson-agx-whisper", "deviceId": "jetson-agx-orin-64gb", "deviceName": "NVIDIA Jetson AGX Orin 64GB", "brand": "NVIDIA", "deviceClass": "edge", "workload": "whisper-large-v3", "audioRtfx": 4.2, "quantization": "FP16", "contextLength": 0, "vramGB": 64, "tdpW": 60, "msrpUsd": 1999, "sourceNote": "whisper.cpp Jetson AGX.", "sourceUrl": "https://github.com/NVIDIA-AI-IOT/jetson-containers", "testedAt": "2024-10-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 644, "isStale": true }, { "id": "bench-x3-h200-embed", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 15200, "quantization": "FP16", "contextLength": 512, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "TEI 1.5 on H200.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-09-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 670, "isStale": true }, { "id": "bench-x3-b200-embed", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 23500, "quantization": "FP16", "contextLength": 512, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "TEI 1.5 on Blackwell DC.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2025-01-18", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 538, "isStale": true }, { "id": "bench-x3-mi300x-embed", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 9800, "quantization": "FP16", "contextLength": 512, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "TEI ROCm MI300X.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference/issues/322", "testedAt": "2024-11-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 587, "isStale": true }, { "id": "bench-x3-l40s-embed", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 10500, "quantization": "FP16", "contextLength": 512, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "TEI L40S.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-08-04", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 705, "isStale": true }, { "id": "bench-x3-a100-80-embed", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 SXM4 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 9500, "quantization": "FP16", "contextLength": 512, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "TEI Ampere DC.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-05-21", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 780, "isStale": true }, { "id": "bench-x3-rtx6000ada-embed", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "embedding-bge-large", "embeddingsPerSec": 8200, "quantization": "FP16", "contextLength": 512, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "TEI Ada pro.", "sourceUrl": "https://github.com/huggingface/text-embeddings-inference", "testedAt": "2024-09-14", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 664, "isStale": true }, { "id": "bench-x3-m3ultra-embed", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "embedding-bge-large", "embeddingsPerSec": 4900, "quantization": "FP16", "contextLength": 512, "vramGB": 512, "tdpW": 100, "msrpUsd": 9999, "sourceNote": "MLX embeddings.", "sourceUrl": "https://huggingface.co/mlx-community", "testedAt": "2024-09-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "freshnessDays": 648, "isStale": true }, { "id": "bench-x3-m4max-deepseek", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 50, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "DeepSeek-R1 Distill 7B MLX.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-02-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000993, "perfPerWatt": 0.769, "tokensPerDollar": 0.011, "freshnessDays": 517, "isStale": true }, { "id": "bench-x3-rx7900xtx-deepseek", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 70, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "DeepSeek-R1 7B ROCm 6.3.", "sourceUrl": "https://github.com/ROCm/ROCm", "testedAt": "2025-02-26", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000151, "perfPerWatt": 0.197, "tokensPerDollar": 0.07, "freshnessDays": 499, "isStale": true }, { "id": "bench-x3-mi300x-deepseek", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 168, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "DeepSeek-R1 distill on MI300X.", "sourceUrl": "https://github.com/vllm-project/vllm/issues/4756", "testedAt": "2025-03-08", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000944, "perfPerWatt": 0.224, "tokensPerDollar": 0.011, "freshnessDays": 489, "isStale": true }, { "id": "bench-x3-h200-deepseek", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 232, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "DeepSeek-R1 7B H200.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-02-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001367, "perfPerWatt": 0.331, "tokensPerDollar": 0.008, "freshnessDays": 513, "isStale": true }, { "id": "bench-x3-b200-deepseek", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 385, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "DeepSeek-R1 Blackwell DC.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/3401", "testedAt": "2025-03-30", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001098, "perfPerWatt": 0.385, "tokensPerDollar": 0.01, "freshnessDays": 467, "isStale": true }, { "id": "bench-x3-l40s-deepseek", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 138, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "DeepSeek-R1 7B L40S.", "sourceUrl": "https://github.com/vllm-project/vllm/discussions/4567", "testedAt": "2025-02-20", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000597, "perfPerWatt": 0.394, "tokensPerDollar": 0.018, "freshnessDays": 505, "isStale": true }, { "id": "bench-x3-rtx5070ti-deepseek", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 105, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "DeepSeek-R1 5070 Ti.", "sourceUrl": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "testedAt": "2025-04-09", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000075, "perfPerWatt": 0.35, "tokensPerDollar": 0.14, "freshnessDays": 457, "isStale": true }, { "id": "bench-x3-rtx5080-deepseek", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 122, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "DeepSeek-R1 on 5080.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1iax2pd/rtx_5080_local_llm/", "testedAt": "2025-03-12", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000087, "perfPerWatt": 0.339, "tokensPerDollar": 0.122, "freshnessDays": 485, "isStale": true }, { "id": "bench-x3-rtx5090-deepseek", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "deepseek-r1-7b-q4", "tokensPerSec": 185, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "DeepSeek-R1 distill 5090.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1iyg6q5/5090_deepseek_r1/", "testedAt": "2025-02-22", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000114, "perfPerWatt": 0.322, "tokensPerDollar": 0.093, "freshnessDays": 503, "isStale": true }, { "id": "bench-x3-rtx4080s-qwen14b", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 68, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Qwen 2.5 14B fits Q4 on 16GB.", "sourceUrl": "https://www.reddit.com/r/LocalLLaMA/comments/1f9p2dx/qwen_25_3090/", "testedAt": "2024-11-13", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000155, "perfPerWatt": 0.213, "tokensPerDollar": 0.068, "freshnessDays": 604, "isStale": true }, { "id": "bench-x3-rtx5080-qwen14b", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 86, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Qwen 2.5 14B 5080.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9521", "testedAt": "2025-03-16", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000123, "perfPerWatt": 0.239, "tokensPerDollar": 0.086, "freshnessDays": 481, "isStale": true }, { "id": "bench-x3-rtx5090-qwen14b", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "qwen2.5-14b-q4", "tokensPerSec": 115, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Qwen 2.5 14B 5090 Blackwell.", "sourceUrl": "https://github.com/ggerganov/llama.cpp/discussions/9521", "testedAt": "2025-03-02", "batchSize": 1, "runsCompleted": 1, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000184, "perfPerWatt": 0.2, "tokensPerDollar": 0.058, "freshnessDays": 495, "isStale": true }, { "id": "bench-asure-rtx5090", "deviceId": "rtx-5090", "deviceName": "NVIDIA GeForce RTX 5090 32GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 85, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 32, "tdpW": 575, "msrpUsd": 1999, "sourceNote": "Trendyol-LLM-Asure 12B Q4 via llama.cpp on Blackwell.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 4.2, "confidence95Low": 80.25, "confidence95High": 89.75, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000249, "perfPerWatt": 0.148, "tokensPerDollar": 0.043, "freshnessDays": 435, "isStale": true, "minObserved": 80, "maxObserved": 90, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8" }, { "id": "bench-asure-rtx4090", "deviceId": "rtx-4090", "deviceName": "NVIDIA GeForce RTX 4090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 60, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 450, "msrpUsd": 1599, "sourceNote": "Trendyol-LLM-Asure 12B Q4 via Ollama.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 3.2, "confidence95Low": 56.38, "confidence95High": 63.62, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000282, "perfPerWatt": 0.133, "tokensPerDollar": 0.038, "freshnessDays": 435, "isStale": true, "minObserved": 56, "maxObserved": 64, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8" }, { "id": "bench-asure-rtx5080", "deviceId": "rtx-5080", "deviceName": "NVIDIA GeForce RTX 5080 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 69.89, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 360, "msrpUsd": 999, "sourceNote": "Trendyol-LLM-Asure 12B Q4 on 5080.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 1.46, "confidence95Low": 68.61, "confidence95High": 71.17, "verificationStatus": "verified-lab", "costPerThousandTokens": 0.000151, "perfPerWatt": 0.194, "tokensPerDollar": 0.07, "freshnessDays": 435, "isStale": true, "minObserved": 67.65, "maxObserved": 71.27, "frameworkVersion": "Ollama 0.24.0 (llama.cpp)", "driverVersion": "595.97", "cudaVersion": "12.8", "osVersion": "Windows 11", "kernelVersion": "10.0.26100", "methodologyVersion": "v2.0", "dockerImage": "ollama/ollama:0.24.0" }, { "id": "bench-asure-rtx4080s", "deviceId": "rtx-4080-super", "deviceName": "NVIDIA GeForce RTX 4080 Super 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 59, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 320, "msrpUsd": 999, "sourceNote": "Trendyol-LLM-Asure 12B fits Q4 on 16GB.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 3, "confidence95Low": 55.61, "confidence95High": 62.39, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000179, "perfPerWatt": 0.184, "tokensPerDollar": 0.059, "freshnessDays": 435, "isStale": true, "minObserved": 55, "maxObserved": 63, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4" }, { "id": "bench-asure-rtx5070ti", "deviceId": "rtx-5070-ti", "deviceName": "NVIDIA GeForce RTX 5070 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 63, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 300, "msrpUsd": 749, "sourceNote": "Trendyol-LLM-Asure 12B 5070 Ti.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 3.1, "confidence95Low": 59.49, "confidence95High": 66.51, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000126, "perfPerWatt": 0.21, "tokensPerDollar": 0.084, "freshnessDays": 435, "isStale": true, "minObserved": 59, "maxObserved": 67, "frameworkVersion": "llama.cpp b4876", "driverVersion": "572.83", "cudaVersion": "12.8" }, { "id": "bench-asure-rtx4070ti", "deviceId": "rtx-4070-ti", "deviceName": "NVIDIA GeForce RTX 4070 Ti 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 44, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 285, "msrpUsd": 799, "sourceNote": "Trendyol-LLM-Asure 12B tight on 12GB.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 2.5, "confidence95Low": 41.17, "confidence95High": 46.83, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000192, "perfPerWatt": 0.154, "tokensPerDollar": 0.055, "freshnessDays": 435, "isStale": true, "minObserved": 41, "maxObserved": 47, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4" }, { "id": "bench-asure-rtx4060ti16", "deviceId": "rtx-4060-ti-16gb", "deviceName": "NVIDIA GeForce RTX 4060 Ti 16GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 28, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 16, "tdpW": 165, "msrpUsd": 499, "sourceNote": "Trendyol-LLM-Asure 12B budget 16GB card.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 1.7, "confidence95Low": 26.08, "confidence95High": 29.92, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000188, "perfPerWatt": 0.17, "tokensPerDollar": 0.056, "freshnessDays": 435, "isStale": true, "minObserved": 26, "maxObserved": 30, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4" }, { "id": "bench-asure-rtx3090", "deviceId": "rtx-3090", "deviceName": "NVIDIA GeForce RTX 3090 24GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 38, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 350, "msrpUsd": 1499, "sourceNote": "Trendyol-LLM-Asure 12B Ampere.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 2.3, "confidence95Low": 35.4, "confidence95High": 40.6, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000417, "perfPerWatt": 0.109, "tokensPerDollar": 0.025, "freshnessDays": 435, "isStale": true, "minObserved": 35, "maxObserved": 41, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4" }, { "id": "bench-asure-rtx3060", "deviceId": "rtx-3060-12gb", "deviceName": "NVIDIA GeForce RTX 3060 12GB", "brand": "NVIDIA", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 20, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 12, "tdpW": 170, "msrpUsd": 329, "sourceNote": "Trendyol-LLM-Asure 12B budget Ampere.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 3, "stdDev": 1.5, "confidence95Low": 18.3, "confidence95High": 21.7, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000174, "perfPerWatt": 0.118, "tokensPerDollar": 0.061, "freshnessDays": 435, "isStale": true, "minObserved": 18, "maxObserved": 22, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4" }, { "id": "bench-asure-h100", "deviceId": "h100-sxm5", "deviceName": "NVIDIA H100 SXM5 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 195, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 700, "msrpUsd": 25000, "sourceNote": "Trendyol-LLM-Asure 12B TRT-LLM on H100.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 7.8, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.001355, "perfPerWatt": 0.279, "tokensPerDollar": 0.008, "freshnessDays": 435, "isStale": true, "minObserved": 185, "maxObserved": 205, "frameworkVersion": "TensorRT-LLM 0.14", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04", "kernelVersion": "5.15.0-124-generic" }, { "id": "bench-asure-h200", "deviceId": "h200", "deviceName": "NVIDIA H200 141GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 230, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 141, "tdpW": 700, "msrpUsd": 30000, "sourceNote": "Trendyol-LLM-Asure 12B H200.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 9.2, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.001379, "perfPerWatt": 0.329, "tokensPerDollar": 0.008, "freshnessDays": 435, "isStale": true, "minObserved": 218, "maxObserved": 242, "frameworkVersion": "TensorRT-LLM 0.14", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04", "kernelVersion": "5.15.0-124-generic" }, { "id": "bench-asure-b200", "deviceId": "b200", "deviceName": "NVIDIA B200 192GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 350, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 1000, "msrpUsd": 39999, "sourceNote": "Trendyol-LLM-Asure 12B on B200.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 4, "stdDev": 10.5, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.001208, "perfPerWatt": 0.35, "tokensPerDollar": 0.009, "freshnessDays": 435, "isStale": true, "minObserved": 336, "maxObserved": 364, "frameworkVersion": "TensorRT-LLM 0.15", "driverVersion": "570.00", "cudaVersion": "12.8", "osVersion": "Ubuntu 22.04", "kernelVersion": "6.5.0-41-generic" }, { "id": "bench-asure-mi300x", "deviceId": "mi300x", "deviceName": "AMD Instinct MI300X 192GB", "brand": "AMD", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 130, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 192, "tdpW": 750, "msrpUsd": 15000, "sourceNote": "Trendyol-LLM-Asure 12B ROCm HIP.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 6.5, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00122, "perfPerWatt": 0.173, "tokensPerDollar": 0.009, "freshnessDays": 435, "isStale": true, "minObserved": 121, "maxObserved": 139, "frameworkVersion": "vLLM 0.6.3 ROCm 6.2", "driverVersion": "6.2.4", "cudaVersion": "N/A (HIP)", "osVersion": "Ubuntu 22.04", "kernelVersion": "5.15.0-91-generic" }, { "id": "bench-asure-m4max", "deviceId": "apple-m4-max", "deviceName": "Apple M4 Max (40c GPU, 128GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 35, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 65, "msrpUsd": 4699, "sourceNote": "Trendyol-LLM-Asure 12B MLX on M4 Max.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 2.5, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001419, "perfPerWatt": 0.538, "tokensPerDollar": 0.007, "freshnessDays": 435, "isStale": true, "minObserved": 32, "maxObserved": 38, "frameworkVersion": "MLX 0.22.0", "driverVersion": "N/A (Metal)", "cudaVersion": "N/A", "osVersion": "macOS 15.1", "kernelVersion": "24.1.0" }, { "id": "bench-asure-m3ultra", "deviceId": "apple-m3-ultra", "deviceName": "Apple M3 Ultra (80c GPU, 512GB)", "brand": "Apple", "deviceClass": "apple-silicon", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 45, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 512, "tdpW": 100, "msrpUsd": 9999, "sourceNote": "Trendyol-LLM-Asure 12B MLX M3 Ultra.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 2.7, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.002349, "perfPerWatt": 0.45, "tokensPerDollar": 0.005, "freshnessDays": 435, "isStale": true, "minObserved": 41, "maxObserved": 49, "frameworkVersion": "MLX 0.22.0", "driverVersion": "N/A (Metal)", "cudaVersion": "N/A", "osVersion": "macOS 15.1", "kernelVersion": "24.1.0" }, { "id": "bench-asure-groq", "deviceId": "groq-lpu", "deviceName": "Groq LPU Inference Engine", "brand": "Groq", "deviceClass": "asic", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 520, "quantization": "FP8", "contextLength": 8192, "vramGB": 230, "tdpW": 215, "msrpUsd": 20000, "sourceNote": "Trendyol-LLM-Asure 12B on Groq LPU.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 4, "stdDev": 15.6, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.000407, "perfPerWatt": 2.419, "tokensPerDollar": 0.026, "freshnessDays": 435, "isStale": true, "minObserved": 498, "maxObserved": 542, "frameworkVersion": "Groq API v1.3", "driverVersion": "N/A (proprietary)", "cudaVersion": "N/A", "osVersion": "N/A (cloud)", "kernelVersion": "N/A" }, { "id": "bench-asure-dgx-spark", "deviceId": "nvidia-dgx-spark", "deviceName": "NVIDIA DGX Spark (Project DIGITS, 128GB)", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 100, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 128, "tdpW": 240, "msrpUsd": 2999, "sourceNote": "Trendyol-LLM-Asure 12B DGX Spark.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 5, "verificationStatus": "vendor-claim", "costPerThousandTokens": 0.000317, "perfPerWatt": 0.417, "tokensPerDollar": 0.033, "freshnessDays": 435, "isStale": true, "minObserved": 93, "maxObserved": 107, "frameworkVersion": "llama.cpp b4876", "driverVersion": "570.00", "cudaVersion": "12.8", "osVersion": "Ubuntu 24.04", "kernelVersion": "6.8.0-52-generic" }, { "id": "bench-asure-l40s", "deviceId": "l40s", "deviceName": "NVIDIA L40S 48GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 110, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 350, "msrpUsd": 7800, "sourceNote": "Trendyol-LLM-Asure 12B L40S.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 5.5, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.00075, "perfPerWatt": 0.314, "tokensPerDollar": 0.014, "freshnessDays": 435, "isStale": true, "minObserved": 102, "maxObserved": 118, "frameworkVersion": "llama.cpp b4876", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04", "kernelVersion": "5.15.0-124-generic" }, { "id": "bench-asure-a100-80", "deviceId": "a100-80gb", "deviceName": "NVIDIA A100 SXM4 80GB", "brand": "NVIDIA", "deviceClass": "datacenter-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 145, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 80, "tdpW": 400, "msrpUsd": 15000, "sourceNote": "Trendyol-LLM-Asure 12B on A100.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 5.8, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.001093, "perfPerWatt": 0.363, "tokensPerDollar": 0.01, "freshnessDays": 435, "isStale": true, "minObserved": 137, "maxObserved": 153, "frameworkVersion": "llama.cpp b4876", "driverVersion": "550.90", "cudaVersion": "12.6", "osVersion": "Ubuntu 22.04", "kernelVersion": "5.15.0-124-generic" }, { "id": "bench-asure-rtx6000ada", "deviceId": "rtx-6000-ada", "deviceName": "NVIDIA RTX 6000 Ada 48GB", "brand": "NVIDIA", "deviceClass": "pro-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 95, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 48, "tdpW": 300, "msrpUsd": 6800, "sourceNote": "Trendyol-LLM-Asure 12B RTX 6000 Ada.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 4.8, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000757, "perfPerWatt": 0.317, "tokensPerDollar": 0.014, "freshnessDays": 435, "isStale": true, "minObserved": 88, "maxObserved": 102, "frameworkVersion": "llama.cpp b4876", "driverVersion": "552.44", "cudaVersion": "12.4", "osVersion": "Ubuntu 24.04", "kernelVersion": "6.8.0-52-generic" }, { "id": "bench-asure-rx7900xtx", "deviceId": "rx-7900-xtx", "deviceName": "AMD Radeon RX 7900 XTX 24GB", "brand": "AMD", "deviceClass": "consumer-gpu", "workload": "trendyol-llm-asure-12b-q4", "tokensPerSec": 52, "quantization": "Q4_K_M", "contextLength": 8192, "vramGB": 24, "tdpW": 355, "msrpUsd": 999, "sourceNote": "Trendyol-LLM-Asure 12B ROCm 6.2.", "sourceUrl": "https://huggingface.co/alibayram/Trendyol-LLM-Asure-12B", "testedAt": "2025-05-01", "batchSize": 1, "runsCompleted": 5, "stdDev": 3.6, "verificationStatus": "curated-aggregate", "costPerThousandTokens": 0.000203, "perfPerWatt": 0.146, "tokensPerDollar": 0.052, "freshnessDays": 435, "isStale": true, "minObserved": 47, "maxObserved": 57, "frameworkVersion": "llama.cpp b4876 ROCm 6.2", "driverVersion": "6.2.4", "cudaVersion": "N/A (HIP)", "osVersion": "Ubuntu 22.04", "kernelVersion": "5.15.0-91-generic" } ]