Inter-rater reliability (IRR) is a measure of consistency between two or more raters. It quantifies the degree to which different observers agree on their judgments. This is crucial in fields like psychology, medicine, and education where subjective assessments are common. High IRR indicates that the measurement tool or criteria are well-defined and consistently applied. Low IRR suggests ambiguity in the criteria or rater training issues.
Inter-Rater Agreement (Cohen's Kappa):
#interRaterAgreementCalculator {
font-family: sans-serif;
border: 1px solid #ccc;
padding: 20px;
border-radius: 8px;
max-width: 500px;
margin: 20px auto;
background-color: #f9f9f9;
}
.input-section {
margin-bottom: 15px;
display: flex;
align-items: center;
gap: 10px;
}
.input-section label {
flex: 1;
text-align: right;
font-weight: bold;
}
.input-section input {
flex: 1;
padding: 8px;
border: 1px solid #ccc;
border-radius: 4px;
box-sizing: border-box;
}
button {
display: block;
width: 100%;
padding: 10px 15px;
background-color: #4CAF50;
color: white;
border: none;
border-radius: 5px;
cursor: pointer;
font-size: 16px;
margin-top: 15px;
}
button:hover {
background-color: #45a049;
}
.result-section {
margin-top: 20px;
padding: 15px;
background-color: #e8f5e9;
border: 1px solid #a5d6a7;
border-radius: 5px;
}
.result-section h3 {
margin-top: 0;
color: #2e7d32;
}
#irrResultValue {
font-size: 24px;
font-weight: bold;
color: #1b5e20;
margin-bottom: 10px;
}
#irrInterpretation {
font-style: italic;
color: #388e3c;
}
function calculateIRR() {
var totalItems = parseFloat(document.getElementById("totalItems").value);
var rater1Agreements = parseFloat(document.getElementById("rater1Agreements").value);
var rater2Agreements = parseFloat(document.getElementById("rater2Agreements").value); // Assuming this is meant to be the same as rater1Agreements for simplicity or a typo
// Basic validation
if (isNaN(totalItems) || isNaN(rater1Agreements) || isNaN(rater2Agreements)) {
document.getElementById("irrResultValue").innerHTML = "Invalid input. Please enter numbers.";
document.getElementById("irrInterpretation").innerHTML = "";
return;
}
if (totalItems <= 0 || rater1Agreements < 0 || rater2Agreements totalItems || rater2Agreements > totalItems) {
document.getElementById("irrResultValue").innerHTML = "Invalid input values.";
document.getElementById("irrInterpretation").innerHTML = "";
return;
}
// For Cohen's Kappa, we need the number of agreements. If rater1Agreements and rater2Agreements are the same, it implies a single count of agreement.
// Let's use the rater1Agreements as the 'observed agreement' (Po).
var observedAgreement = rater1Agreements;
// Calculate expected agreement (Pe)
// This requires the marginal distributions (how many items each rater scored in each category).
// For a simple calculator, we often assume binary categories (e.g., agreement/disagreement) and can't calculate true Cohen's Kappa without more detailed data.
// However, a common simplification or a different metric might be expected.
// A simple "percent agreement" is often used as a proxy or a starting point.
// Let's calculate Percent Agreement first, as true Cohen's Kappa requires marginals.
var percentAgreement = (observedAgreement / totalItems) * 100;
// If the intention was for a *very* simplified IRR, percent agreement might suffice.
// If Cohen's Kappa is strictly required, we need more input fields for the marginal counts.
// For a more robust solution, Cohen's Kappa requires:
// – Number of items where Rater 1 said Yes, Rater 2 said Yes (a)
// – Number of items where Rater 1 said Yes, Rater 2 said No (b)
// – Number of items where Rater 1 said No, Rater 2 said Yes (c)
// – Number of items where Rater 1 said No, Rater 2 said No (d)
// Where a + b + c + d = totalItems.
// Observed Agreement (Po) = (a + d) / totalItems
// Expected Agreement (Pe) = ((a+b)*(a+c) + (b+d)*(c+d)) / totalItems^2
// Kappa = (Po – Pe) / (1 – Pe)
// Since we only have total items and total agreements, we can only calculate Percent Agreement.
// We'll present Percent Agreement and mention the limitation.
var resultDisplay = percentAgreement.toFixed(2) + "%";
var interpretation = "";
if (percentAgreement >= 80) {
interpretation = "Excellent agreement.";
} else if (percentAgreement >= 60) {
interpretation = "Good agreement.";
} else if (percentAgreement >= 40) {
interpretation = "Fair agreement.";
} else {
interpretation = "Poor agreement.";
}
document.getElementById("irrResultValue").innerHTML = resultDisplay;
document.getElementById("irrInterpretation").innerHTML = "Interpretation: " + interpretation + "Note: This calculator provides Percent Agreement. For Cohen's Kappa, additional data on disagreement categories is required.";
}