Determine if your test results are statistically valid
Control Group (A)
Variant Group (B)
Confidence Level
Relative Improvement
P-Value
Z-Score
Understanding Statistical Significance
In A/B testing, statistical significance is the probability that the difference in conversion rates between your control and variant is not due to random chance. This calculator uses a two-tailed Z-test to evaluate the data provided.
What is a P-Value?
A p-value measures the evidence against the null hypothesis (which assumes there is no difference between groups). A p-value of 0.05 or lower typically suggests that the result is "statistically significant," meaning there is a 95% confidence level that the variation caused the change.
How to Read the Results
Confidence Level > 95%: Generally considered a safe winner. You can roll out the change.
Confidence Level 90-95%: Suggestive but may require more data to be certain.
Confidence Level < 90%: Not significant. The difference is likely due to noise or random fluctuations.
Calculation Example
If Group A has 1,000 visitors and 50 conversions (5%), and Group B has 1,000 visitors and 70 conversions (7%), the relative improvement is 40%. The resulting p-value is approximately 0.057, giving a 94.3% confidence level. While strong, many marketers wait for 95% before declaring a definitive winner.
function calculateStatSig() {
var cV = parseFloat(document.getElementById('controlVisitors').value);
var cC = parseFloat(document.getElementById('controlConversions').value);
var vV = parseFloat(document.getElementById('variantVisitors').value);
var vC = parseFloat(document.getElementById('variantConversions').value);
if (!cV || !cC || !vV || !vC || cV <= 0 || vV <= 0 || cC < 0 || vC < 0) {
alert("Please enter valid positive numbers for all fields.");
return;
}
var controlRate = cC / cV;
var variantRate = vC / vV;
var improvement = ((variantRate – controlRate) / controlRate) * 100;
// Pooled proportion
var pPooled = (cC + vC) / (cV + vV);
// Standard Error
var se = Math.sqrt(pPooled * (1 – pPooled) * (1/cV + 1/vV));
// Z-Score
var zScore = (variantRate – controlRate) / se;
// P-Value approximation (Error Function)
function erf(x) {
var a1 = 0.254829592;
var a2 = -0.284496736;
var a3 = 1.421413741;
var a4 = -1.453152027;
var a5 = 1.061405429;
var p = 0.3275911;
var sign = 1;
if (x 0 ? '+' : ") + improvement.toFixed(2) + '%';
document.getElementById('pValueResult').innerText = pValue.toFixed(4);
document.getElementById('zScoreResult').innerText = zScore.toFixed(4);
if (confidence >= 95) {
verdictBox.style.background = '#d4edda';
verdictBox.style.color = '#155724';
verdictBox.innerText = "Significant Result! (Winner Found)";
} else if (confidence >= 90) {
verdictBox.style.background = '#fff3cd';
verdictBox.style.color = '#856404';
verdictBox.innerText = "Trending Positive (Needs More Data)";
} else {
verdictBox.style.background = '#f8d7da';
verdictBox.style.color = '#721c24';
verdictBox.innerText = "Not Statistically Significant";
}
}