[ { "id": "llama-3.2-3b", "name": "Llama 3.2 3B", "family": "Llama", "params_b": 3.21, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 28, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": false, "architecture_source_url": "https://huggingface.co/unsloth/Llama-3.2-3B-Instruct/raw/main/config.json", "notes": "Small enough for a Mac mini or a modest GPU. A good stack to test your setup.", "community_measurement": null }, { "id": "llama-3.1-8b", "name": "Llama 3.1 8B", "family": "Llama", "params_b": 8.03, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 32, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": false, "architecture_source_url": "https://huggingface.co/unsloth/Meta-Llama-3.1-8B-Instruct/raw/main/config.json", "notes": "The default local model. Q4 fits in 8–12 GB, Q8 wants closer to 12–16 GB.", "community_measurement": null }, { "id": "mistral-7b-v03", "name": "Mistral 7B v0.3", "family": "Mistral", "params_b": 7.25, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 32, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 32768, "architecture_estimated": true, "architecture_source_url": null, "notes": "Architecture is an estimate of the familiar 7B shape. No config was fetched for this dataset.", "community_measurement": null }, { "id": "gemma-2-9b", "name": "Gemma 2 9B", "family": "Gemma", "params_b": 9.24, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 42, "kv_heads": 8, "head_dim": 256, "full_attention_fraction": 0.5, "sliding_window": 4096, "max_context": 8192, "architecture_estimated": true, "architecture_source_url": null, "notes": "Gemma 2 alternates local and global attention. The KV cache is an approximation, not a measured file.", "community_measurement": null }, { "id": "qwen2.5-14b", "name": "Qwen2.5 14B", "family": "Qwen", "params_b": 14.7, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 48, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": false, "architecture_source_url": "https://huggingface.co/Qwen/Qwen2.5-14B/raw/main/config.json", "notes": "A strong mid-size model. Q4 needs about 10 GB plus context. Sliding window is off in the config.", "community_measurement": null }, { "id": "phi-4", "name": "Phi-4 14B", "family": "Phi", "params_b": 14.7, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 40, "kv_heads": 10, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 16384, "architecture_estimated": true, "architecture_source_url": null, "notes": "Architecture figures are an estimate. No config was fetched, so the VRAM line can be slightly off.", "community_measurement": null }, { "id": "gemma-2-27b", "name": "Gemma 2 27B", "family": "Gemma", "params_b": 27.2, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 46, "kv_heads": 16, "head_dim": 128, "full_attention_fraction": 0.5, "sliding_window": 4096, "max_context": 8192, "architecture_estimated": true, "architecture_source_url": null, "notes": "Same caveat as Gemma 2 9B: sliding window makes the KV estimate rougher.", "community_measurement": null }, { "id": "qwen2.5-32b", "name": "Qwen2.5 32B", "family": "Qwen", "params_b": 32.8, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 64, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": false, "architecture_source_url": "https://huggingface.co/Qwen/Qwen2.5-32B/raw/main/config.json", "notes": "Q4 wants a bit over 20 GB. A 16 GB card drops out; a 24 GB card is tight.", "community_measurement": null }, { "id": "llama-3.3-70b", "name": "Llama 3.3 70B", "family": "Llama", "params_b": 70.6, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 80, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": false, "architecture_source_url": "https://huggingface.co/unsloth/Llama-3.3-70B-Instruct/raw/main/config.json", "notes": "Q4 means Q4_K_M. The 0.60 formula is about 42 GB of weights. A bartowski Llama-3.3-70B-Instruct Q4_K_M file is 42.52 GB (42,520,398,816 bytes). A single 24 GB or 32 GB card is not enough.", "community_measurement": null }, { "id": "qwen2.5-72b", "name": "Qwen2.5 72B", "family": "Qwen", "params_b": 72.7, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 80, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": false, "architecture_source_url": "https://huggingface.co/Qwen/Qwen2.5-72B/raw/main/config.json", "notes": "Same class as Llama 70B. A bartowski Qwen2.5-72B-Instruct Q4_K_M file is 47.42 GB (47,415,715,488 bytes), larger than 72.7 × 0.60. IQ4_XS from the same repo is 39.71 GB. You need a large memory pool.", "community_measurement": null }, { "id": "deepseek-r1-distill-70b", "name": "DeepSeek-R1-Distill 70B", "family": "DeepSeek", "params_b": 70.6, "active_params_b": null, "params_estimated": true, "params_source_url": null, "layers": 80, "kv_heads": 8, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 131072, "architecture_estimated": true, "architecture_source_url": null, "notes": "A distill on a Llama 70B shape. Architecture is copied from Llama 3.3 70B and marked as an estimate: no separate config was fetched.", "community_measurement": null }, { "id": "nemotron-3-5-bolt-74b", "name": "NVIDIA Nemotron 3.5 Bolt 74B-A9B", "family": "Nemotron", "params_b": 74, "active_params_b": 9, "params_estimated": true, "params_source_url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "layers": 8, "kv_heads": 2, "head_dim": 128, "full_attention_fraction": 1, "sliding_window": null, "max_context": 262144, "architecture_estimated": true, "architecture_source_url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB/raw/main/config.json", "notes": "Hugging Face model card, cited as 10/15/2026: 74B total, 9B active, OpenMDW-1.1, context up to 256K. VRAM math uses the 74B total. Catalog Q4, Q5, Q8 and FP16 rates are not the published 2-bit pack. The config's language stack is a hybrid of 40 Mamba, 40 MoE and 8 attention blocks; the KV term uses only those 8 attention blocks and is not NVIDIA's cache.", "community_measurement": { "label": "community measurement, single run", "source_name": "llm-speed", "source_url": "https://llm-speed.com/blog/nemotron-3-5-bolt-74b-24gb-measured", "published": "2026-10-16", "note": "Published 16 Oct 2026. NVIDIA official 2-bit GGUF, stated file size 22.0 GB. llama.cpp 0.7.0-dev build 11669, commit 620d2225c, 4K context, chat-short, one run per config, runs dated 15 Oct 2026. Headline figures are 87.8, 107.3 and 154.7 tokens per second. The table prints 87.75 for the RTX 3090 and says the quant was not reported in that column. Not a localLLMrig measurement. Calculator estimates keep benchmarkSourceUrl null.", "rows": [ { "hardware": "RTX 3090", "tokens_per_sec": 87.8 }, { "hardware": "RTX 4090", "tokens_per_sec": 107.3 }, { "hardware": "RTX 5090", "tokens_per_sec": 154.7 } ] } } ]