{"success":true,"version":"v1","request":{"tool":"get_calculator_schema","calculator_id":"ab-test-significance"},"result":{"entity_type":"calculator","id":"ab-test-significance","calculator_id":"ab-test-significance","canonical_url":"https://tttkmbb.com/business/ab-test-significance","name":"A/B Test Significance Calculator","title":"A/B Test Significance Calculator – Two-Proportion Z-Test, P-Value, Lift and Confidence Interval","category":"business","category_name":"Business & Marketing","tool_name":"calculate_ab_test_significance","featured_mcp_tool":true,"description":"Tests whether a variant's conversion rate differs from the control's using the pooled two-proportion z-test, and reports both rates, absolute and relative lift, z-score, two-sided p-value, a confidence interval for the difference and the sample size needed per variant to detect the observed lift with 80 % power.","use_when":"You have visitors and conversions for a control and a variant (A/B test, split test, email test) and need to know whether the difference is statistically significant.","do_not_use_when":"The metric is a mean (revenue per user, time on page) rather than a conversion proportion, or there are more than two variants (use a chi-square or multiple-comparison procedure).","inputs":[{"name":"control_visitors","label":"Control visitors","type":"integer","required":true,"max":1000000000000,"exclusive_min":0,"description":"Number of users exposed to the control (A).","example":1000},{"name":"control_conversions","label":"Control conversions","type":"integer","required":true,"min":0,"max":1000000000000,"description":"Conversions in the control group.","example":50},{"name":"variant_visitors","label":"Variant visitors","type":"integer","required":true,"max":1000000000000,"exclusive_min":0,"description":"Number of users exposed to the variant (B).","example":1000},{"name":"variant_conversions","label":"Variant conversions","type":"integer","required":true,"min":0,"max":1000000000000,"description":"Conversions in the variant group.","example":70},{"name":"confidence_level_percent","label":"Confidence level","type":"number","unit":"%","required":false,"default":95,"exclusive_min":0,"exclusive_max":100,"description":"Confidence level for the significance decision and interval (95 → α = 0.05, two-sided).","example":95}],"outputs":[{"name":"control_rate_percent","label":"Control conversion rate","type":"number","unit":"%","decimals":2,"description":"control_conversions / control_visitors × 100."},{"name":"variant_rate_percent","label":"Variant conversion rate","type":"number","unit":"%","decimals":2,"description":"variant_conversions / variant_visitors × 100."},{"name":"absolute_lift_percentage_points","label":"Absolute lift","type":"number","unit":"pp","decimals":2,"description":"variant rate − control rate, in percentage points."},{"name":"relative_lift_percent","label":"Relative lift","type":"number","unit":"%","decimals":2,"description":"(variant rate − control rate) / control rate × 100."},{"name":"standard_error_percentage_points","label":"Pooled standard error","type":"number","unit":"pp","decimals":4,"description":"√(p̄(1 − p̄)(1/n1 + 1/n2)) × 100 under the null hypothesis."},{"name":"z_score","label":"z-score","type":"number","decimals":4,"description":"(p2 − p1) / pooled standard error; positive when the variant is higher."},{"name":"p_value","label":"p-value (two-sided)","type":"number","decimals":6,"description":"Probability of a difference at least this large if both rates were equal."},{"name":"significant","label":"Significant","type":"boolean","decimals":4,"description":"true when p_value < 1 − confidence_level/100."},{"name":"ci_lower_percentage_points","label":"CI lower (difference)","type":"number","unit":"pp","decimals":2,"description":"Lower bound of the confidence interval for variant − control (unpooled Wald interval)."},{"name":"ci_upper_percentage_points","label":"CI upper (difference)","type":"number","unit":"pp","decimals":2,"description":"Upper bound of the same interval."},{"name":"required_visitors_per_variant","label":"Sample size per variant (80 % power)","type":"integer","decimals":4,"description":"Visitors per group needed to detect the observed difference with 80 % power at this confidence level (rounded up)."},{"name":"verdict","label":"Verdict","type":"string","decimals":4,"description":"Plain-language conclusion."}],"input_schema":{"type":"object","properties":{"control_visitors":{"description":"Number of users exposed to the control (A).","type":"integer","maximum":1000000000000,"exclusiveMinimum":0,"examples":[1000]},"control_conversions":{"description":"Conversions in the control group.","type":"integer","minimum":0,"maximum":1000000000000,"examples":[50]},"variant_visitors":{"description":"Number of users exposed to the variant (B).","type":"integer","maximum":1000000000000,"exclusiveMinimum":0,"examples":[1000]},"variant_conversions":{"description":"Conversions in the variant group.","type":"integer","minimum":0,"maximum":1000000000000,"examples":[70]},"confidence_level_percent":{"description":"Confidence level for the significance decision and interval (95 → α = 0.05, two-sided). Unit: %.","type":"number","exclusiveMinimum":0,"exclusiveMaximum":100,"default":95,"examples":[95],"x-unit":"%"}},"additionalProperties":false,"required":["control_visitors","control_conversions","variant_visitors","variant_conversions"]},"output_schema":{"type":"object","properties":{"control_rate_percent":{"description":"control_conversions / control_visitors × 100. Unit: %.","type":"number","x-unit":"%"},"variant_rate_percent":{"description":"variant_conversions / variant_visitors × 100. Unit: %.","type":"number","x-unit":"%"},"absolute_lift_percentage_points":{"description":"variant rate − control rate, in percentage points. Unit: pp.","type":"number","x-unit":"pp"},"relative_lift_percent":{"description":"(variant rate − control rate) / control rate × 100. Unit: %.","type":"number","x-unit":"%"},"standard_error_percentage_points":{"description":"√(p̄(1 − p̄)(1/n1 + 1/n2)) × 100 under the null hypothesis. Unit: pp.","type":"number","x-unit":"pp"},"z_score":{"description":"(p2 − p1) / pooled standard error; positive when the variant is higher.","type":"number"},"p_value":{"description":"Probability of a difference at least this large if both rates were equal.","type":"number"},"significant":{"description":"true when p_value < 1 − confidence_level/100.","type":"boolean"},"ci_lower_percentage_points":{"description":"Lower bound of the confidence interval for variant − control (unpooled Wald interval). Unit: pp.","type":"number","x-unit":"pp"},"ci_upper_percentage_points":{"description":"Upper bound of the same interval. Unit: pp.","type":"number","x-unit":"pp"},"required_visitors_per_variant":{"description":"Visitors per group needed to detect the observed difference with 80 % power at this confidence level (rounded up).","type":"integer"},"verdict":{"description":"Plain-language conclusion.","type":"string"}}},"formula":"p1 = x1/n1, p2 = x2/n2, p̄ = (x1 + x2)/(n1 + n2); z = (p2 − p1) / √(p̄(1 − p̄)(1/n1 + 1/n2)); p_value = 2·(1 − Φ(|z|)); CI = (p2 − p1) ± z_crit·√(p1(1 − p1)/n1 + p2(1 − p2)/n2); n per variant = (z_crit·√(2·p̄(1 − p̄)) + z_0.80·√(p1(1 − p1) + p2(1 − p2)))² / (p2 − p1)²","method":"Pooled two-proportion z-test (normal approximation; adequate when each group has at least about 10 conversions and 10 non-conversions). The sample-size formula is the standard two-sample proportion formula (Fleiss) without continuity correction, with p̄ the mean of the two rates.","sources":[{"name":"NIST/SEMATECH e-Handbook of Statistical Methods, 7.3.3 Do two processes produce the same proportion of defectives?","url":"https://www.itl.nist.gov/div898/handbook/prc/section3/prc33.htm","type":"government","retrieved_at":"2026-09-24"},{"name":"Wikipedia – Two-proportion Z-test","url":"https://en.wikipedia.org/wiki/Two-proportion_Z-test","type":"reference","retrieved_at":"2026-09-24"},{"name":"Kohavi R, Tang D, Xu Y (2020) Trustworthy Online Controlled Experiments, Cambridge University Press","url":"https://www.cambridge.org/core/books/trustworthy-online-controlled-experiments/D97B26382EB0EB2DC2019A7A7B518F59","type":"textbook","retrieved_at":"2026-09-24"}],"freshness":{"type":"static","max_age_seconds":null,"note":"Deterministic formula with fixed constants; results never go stale. Inputs supplied by the caller determine the output."},"examples":[{"name":"1,000/50 vs 1,000/70 at 95 %","inputs":{"control_visitors":1000,"control_conversions":50,"variant_visitors":1000,"variant_conversions":70},"expected":{"control_rate_percent":5,"variant_rate_percent":7,"absolute_lift_percentage_points":2,"relative_lift_percent":40,"standard_error_percentage_points":1.0621,"z_score":1.8831,"p_value":0.0597,"significant":false,"ci_lower_percentage_points":-0.08,"ci_upper_percentage_points":4.08,"required_visitors_per_variant":2213},"url":"https://tttkmbb.com/api/v1/calculate/ab-test-significance?control_visitors=1000&control_conversions=50&variant_visitors=1000&variant_conversions=70"},{"name":"5,000/200 vs 5,000/260 at 95 %","inputs":{"control_visitors":5000,"control_conversions":200,"variant_visitors":5000,"variant_conversions":260},"expected":{"control_rate_percent":4,"variant_rate_percent":5.2,"relative_lift_percent":30,"z_score":2.8642,"p_value":0.0042,"significant":true,"ci_lower_percentage_points":0.38,"ci_upper_percentage_points":2.02,"required_visitors_per_variant":4783},"url":"https://tttkmbb.com/api/v1/calculate/ab-test-significance?control_visitors=5000&control_conversions=200&variant_visitors=5000&variant_conversions=260"}],"faq":[{"q":"Why is a 40 % lift not significant?","a":"With 1,000 visitors per group the rates 5 % and 7 % have a pooled standard error of about 1.06 points, so the 2-point difference is only 1.88 standard errors away from zero (p = 0.06). About 2,213 visitors per group would be needed to detect it reliably."},{"q":"One-sided or two-sided?","a":"Two-sided: it tests for a difference in either direction. For a one-sided test (variant better only) halve the p-value."},{"q":"Does the confidence interval match the p-value?","a":"Approximately: the interval uses the unpooled standard error while the test uses the pooled one, so near the threshold they can disagree slightly."}],"tags":["ab test calculator","a/b test significance","split test","two proportion z test","p value","conversion rate test"],"related":[{"calculator_id":"conversion-rate","reason":"Conversion rate of a single variant."},{"calculator_id":"sample-size","reason":"Plan a sample size for a target margin of error."},{"calculator_id":"normal-distribution","reason":"Look up the normal probabilities behind the z-score."}],"links":{"html":"https://tttkmbb.com/business/ab-test-significance","markdown":"https://tttkmbb.com/business/ab-test-significance.md","json":"https://tttkmbb.com/business/ab-test-significance.json","api":"https://tttkmbb.com/api/v1/calculate/ab-test-significance","schema":"https://tttkmbb.com/api/v1/calculators/ab-test-significance","openapi":"https://tttkmbb.com/openapi.json","mcp":"https://tttkmbb.com/mcp"},"version":"v1","updated_at":"2026-09-24"},"timestamp":"2026-09-24T01:50:28Z","next_actions":[{"tool":"calculate_ab_test_significance","calculator_id":"ab-test-significance","reason":"Run A/B Test Significance Calculator with the inputs above."}],"links":{"markdown":"https://tttkmbb.com/business/ab-test-significance.md"}}