{"success":true,"version":"v1","request":{"tool":"get_calculator_schema","calculator_id":"llm-vram-estimate"},"result":{"entity_type":"calculator","id":"llm-vram-estimate","calculator_id":"llm-vram-estimate","canonical_url":"https://tttkmbb.com/developer/llm-vram-estimate","name":"LLM VRAM Calculator","title":"LLM VRAM Calculator – GPU Memory for Model Weights, KV Cache and Overhead by Parameter Count and Quantization","category":"developer","category_name":"Developer & IT","tool_name":"estimate_llm_vram","featured_mcp_tool":false,"description":"Estimates the GPU memory needed to run a transformer language model for inference from its parameter count and bits per weight (weights), layer count, hidden size, context length and batch size (KV cache), plus a percentage overhead for activations and the CUDA context, and lists which common GPU sizes it fits.","use_when":"You want to know whether a model of N billion parameters fits a given GPU at 4-, 8- or 16-bit precision, or how much the KV cache grows with context length.","do_not_use_when":"You are sizing training memory (optimizer states and gradients need roughly 3–4× the weight memory in mixed precision) or the model uses mixture-of-experts offloading; the estimate ignores framework-specific allocators.","inputs":[{"name":"parameters_billions","label":"Parameters","type":"number","unit":"billion","required":true,"max":100000,"exclusive_min":0,"description":"Model size in billions of parameters (7 for a 7B model).","example":7},{"name":"bits_per_weight","label":"Bits per weight","type":"number","required":false,"default":16,"min":1,"max":64,"description":"Weight precision: 16 for FP16/BF16, 8 for INT8/FP8, 4 for 4-bit quantization (GPTQ/AWQ/Q4), 32 for FP32; fractional values such as 4.5 (Q4_K_M) are allowed.","example":16},{"name":"context_tokens","label":"Context length","type":"integer","unit":"tokens","required":false,"default":4096,"min":1,"max":10000000,"description":"Tokens held in the KV cache per sequence (prompt + generated).","example":4096},{"name":"batch_size","label":"Batch size","type":"integer","required":false,"default":1,"min":1,"max":4096,"description":"Concurrent sequences.","example":1},{"name":"num_layers","label":"Transformer layers","type":"integer","required":false,"min":1,"max":1000,"description":"Number of decoder layers (needed for the KV cache).","example":32},{"name":"hidden_size","label":"Hidden size","type":"integer","required":false,"min":1,"max":100000,"description":"Model dimension d_model; for grouped-query attention use num_kv_heads × head_dim instead.","example":4096},{"name":"kv_bits","label":"KV cache bits","type":"number","required":false,"default":16,"min":1,"max":32,"description":"Precision of the cached keys and values (16 for FP16, 8 for FP8/INT8 cache).","example":16},{"name":"overhead_percent","label":"Overhead","type":"number","unit":"%","required":false,"default":20,"min":0,"max":500,"description":"Extra memory for activations, CUDA context and allocator fragmentation, as a percentage of weights + KV cache.","example":20}],"outputs":[{"name":"weights_gb","label":"Weights","type":"number","unit":"GB","decimals":3,"description":"parameters × bits_per_weight / 8, in gigabytes (10⁹ bytes)."},{"name":"kv_cache_gb","label":"KV cache","type":"number","unit":"GB","decimals":3,"description":"2 × layers × hidden_size × context_tokens × batch_size × kv_bits / 8; 0 when layers or hidden size are not given."},{"name":"overhead_gb","label":"Overhead","type":"number","unit":"GB","decimals":3,"description":"(weights + KV cache) × overhead_percent / 100."},{"name":"total_gb","label":"Total","type":"number","unit":"GB","decimals":3,"description":"weights + KV cache + overhead, gigabytes (10⁹ bytes)."},{"name":"total_gib","label":"Total (GiB)","type":"number","unit":"GiB","decimals":3,"description":"Total in gibibytes (2³⁰ bytes), the unit GPU memory is actually specified in."},{"name":"kv_cache_per_token_kb","label":"KV cache per token","type":"number","unit":"kB","decimals":3,"description":"2 × layers × hidden_size × kv_bits / 8 per token, in kilobytes."},{"name":"fits_gpu","label":"Fits on","type":"string","decimals":4,"description":"Which of the common 8 / 12 / 16 / 24 / 32 / 48 / 80 GB (GiB) single cards can hold the total."},{"name":"gpus_80gb_needed","label":"80 GB GPUs needed","type":"integer","decimals":4,"description":"ceil(total_gib / 80): number of 80 GB accelerators for tensor-parallel serving, ignoring communication buffers."}],"input_schema":{"type":"object","properties":{"parameters_billions":{"description":"Model size in billions of parameters (7 for a 7B model). Unit: billion.","type":"number","maximum":100000,"exclusiveMinimum":0,"examples":[7],"x-unit":"billion"},"bits_per_weight":{"description":"Weight precision: 16 for FP16/BF16, 8 for INT8/FP8, 4 for 4-bit quantization (GPTQ/AWQ/Q4), 32 for FP32; fractional values such as 4.5 (Q4_K_M) are allowed.","type":"number","minimum":1,"maximum":64,"default":16,"examples":[16]},"context_tokens":{"description":"Tokens held in the KV cache per sequence (prompt + generated). Unit: tokens.","type":"integer","minimum":1,"maximum":10000000,"default":4096,"examples":[4096],"x-unit":"tokens"},"batch_size":{"description":"Concurrent sequences.","type":"integer","minimum":1,"maximum":4096,"default":1,"examples":[1]},"num_layers":{"description":"Number of decoder layers (needed for the KV cache).","type":"integer","minimum":1,"maximum":1000,"examples":[32]},"hidden_size":{"description":"Model dimension d_model; for grouped-query attention use num_kv_heads × head_dim instead.","type":"integer","minimum":1,"maximum":100000,"examples":[4096]},"kv_bits":{"description":"Precision of the cached keys and values (16 for FP16, 8 for FP8/INT8 cache).","type":"number","minimum":1,"maximum":32,"default":16,"examples":[16]},"overhead_percent":{"description":"Extra memory for activations, CUDA context and allocator fragmentation, as a percentage of weights + KV cache. Unit: %.","type":"number","minimum":0,"maximum":500,"default":20,"examples":[20],"x-unit":"%"}},"additionalProperties":false,"required":["parameters_billions"]},"output_schema":{"type":"object","properties":{"weights_gb":{"description":"parameters × bits_per_weight / 8, in gigabytes (10⁹ bytes). Unit: GB.","type":"number","x-unit":"GB"},"kv_cache_gb":{"description":"2 × layers × hidden_size × context_tokens × batch_size × kv_bits / 8; 0 when layers or hidden size are not given. Unit: GB.","type":"number","x-unit":"GB"},"overhead_gb":{"description":"(weights + KV cache) × overhead_percent / 100. Unit: GB.","type":"number","x-unit":"GB"},"total_gb":{"description":"weights + KV cache + overhead, gigabytes (10⁹ bytes). Unit: GB.","type":"number","x-unit":"GB"},"total_gib":{"description":"Total in gibibytes (2³⁰ bytes), the unit GPU memory is actually specified in. Unit: GiB.","type":"number","x-unit":"GiB"},"kv_cache_per_token_kb":{"description":"2 × layers × hidden_size × kv_bits / 8 per token, in kilobytes. Unit: kB.","type":"number","x-unit":"kB"},"fits_gpu":{"description":"Which of the common 8 / 12 / 16 / 24 / 32 / 48 / 80 GB (GiB) single cards can hold the total.","type":"string"},"gpus_80gb_needed":{"description":"ceil(total_gib / 80): number of 80 GB accelerators for tensor-parallel serving, ignoring communication buffers.","type":"integer"}}},"formula":"weights_gb = parameters_billions × bits_per_weight / 8; kv_cache_gb = 2 × num_layers × hidden_size × context_tokens × batch_size × kv_bits / 8 / 10⁹; overhead_gb = (weights_gb + kv_cache_gb) × overhead_percent / 100; total_gb = weights_gb + kv_cache_gb + overhead_gb","method":"Standard inference sizing: weights scale with parameters × precision, the KV cache stores one key and one value vector per layer per token (multi-head attention; for GQA models pass num_kv_heads × head_dim as hidden_size). Embedding/vocabulary matrices are included in the parameter count; quantized models keep some tensors at higher precision, so real files are 5–15 % larger than the ideal figure.","sources":[{"name":"Pope et al. (2022) Efficiently Scaling Transformer Inference (arXiv:2211.05102)","url":"https://arxiv.org/abs/2211.05102","type":"peer_reviewed","retrieved_at":"2026-09-24"},{"name":"Kwon et al. (2023) Efficient Memory Management for Large Language Model Serving with PagedAttention (arXiv:2309.06180)","url":"https://arxiv.org/abs/2309.06180","type":"peer_reviewed","retrieved_at":"2026-09-24"}],"freshness":{"type":"static","max_age_seconds":null,"note":"Deterministic formula with fixed constants; results never go stale. Inputs supplied by the caller determine the output."},"examples":[{"name":"7B at FP16 without KV cache detail","inputs":{"parameters_billions":7,"bits_per_weight":16},"expected":{"weights_gb":14,"kv_cache_gb":0,"overhead_gb":2.8,"total_gb":16.8,"total_gib":15.646,"fits_gpu":"16 GB, 24 GB, 32 GB, 48 GB, 80 GB"},"url":"https://tttkmbb.com/api/v1/calculate/llm-vram-estimate?parameters_billions=7&bits_per_weight=16"},{"name":"70B at 4-bit, 80 layers, hidden 8192, 4096 context","inputs":{"parameters_billions":70,"bits_per_weight":4,"num_layers":80,"hidden_size":8192,"context_tokens":4096,"batch_size":1},"expected":{"weights_gb":35,"kv_cache_gb":10.737,"overhead_gb":9.147,"total_gb":54.885,"total_gib":51.116,"gpus_80gb_needed":1},"url":"https://tttkmbb.com/api/v1/calculate/llm-vram-estimate?parameters_billions=70&bits_per_weight=4&num_layers=80&hidden_size=8192&context_tokens=4096&batch_size=1"}],"faq":[{"q":"Why is the KV cache so large for a 70B model?","a":"Every token keeps a key and a value vector in every layer: 2 × 80 layers × 8192 × 2 bytes ≈ 2.6 MB per token, so 4,096 tokens need 10.7 GB. Grouped-query attention (Llama 3 70B: 8 KV heads × 128 = 1024) cuts this eightfold; pass that value as hidden_size."},{"q":"Does a 7B model fit in 8 GB?","a":"At 4 bits the weights are 3.5 GB and the total about 4.2 GB, so yes; at FP16 it needs 14 GB of weights alone."},{"q":"What about training?","a":"Full fine-tuning in mixed precision needs about 16 bytes per parameter (weights, gradients, Adam moments) plus activations; this calculator covers inference only."}],"tags":["llm vram","gpu memory calculator","kv cache","model size","quantization","70b vram"],"related":[{"calculator_id":"llm-token-cost","reason":"Cost of the same model served through a token-priced API."},{"calculator_id":"data-storage","reason":"Convert between GB and GiB."}],"links":{"html":"https://tttkmbb.com/developer/llm-vram-estimate","markdown":"https://tttkmbb.com/developer/llm-vram-estimate.md","json":"https://tttkmbb.com/developer/llm-vram-estimate.json","api":"https://tttkmbb.com/api/v1/calculate/llm-vram-estimate","schema":"https://tttkmbb.com/api/v1/calculators/llm-vram-estimate","openapi":"https://tttkmbb.com/openapi.json","mcp":"https://tttkmbb.com/mcp"},"version":"v1","updated_at":"2026-09-24"},"timestamp":"2026-09-24T03:44:27Z"}