Confidence Intervals

Demonstrating confidence interval coverage using a deck of cards

CI = 80%

Number of cards: 3 5 10 30

M =

#% of # samples contain mu

About

A deck of playing cards is a known population consisting of the numbers 1 through 13 repeated 4 times, once for each suit. The average of this population (\(\mu\)) is 7.00, and the standard deviation (\(\sigma\)) is 3.78.

In the visualization above, you can take samples from this population repeatedly. Each sample produces an estimate of the population parameter with a margin of uncertainty: this is the confidence interval for the sample.

The interpretation of the confidence interval, however, is subtle. The confidence interval is a statement about repeated, long-run probabilities. If we sample from a population repeatedly, the proportion of confidence intervals which include the true population mean will be equal to the specified level of confidence. For example, if we compute an 80% CI for each sample, around 80% of those CIs will include the population mean.

So what does the confidence interval for a single sample tell us? The width of the CI reflects the variability in our sample, but it doesn’t allow us to put a probabilistic value on the population parameter. A common mistake is to think that the confidence interval tells us, with the given degree of confidence, that the true population parameter is a value somewhere within the stated range. This sounds intuitive but it’s not quite true. Under the frequentist approach, the population parameter is a fixed constant, and we can’t make probabilistic statements about constants. So strictly speaking, the CI for a single sample reflects the uncertainty in that sample’s estimate of the parameter rather than uncertainty about the true value of the parameter.

Maybe that sounds a bit disappointing. Surely the whole point of taking a sample is to be able to say something about the population it came from? Well, yes. This is where confidence intervals connect to Null Hypothesis Significance Testing. Rather than saying how confident we are that the population parameter is within a certain range, the sample (and its CI) can help us make an inference about whether the parameter is (or isn’t) some specific hypothetical value.

Let’s say we suspect that someone has removed some cards from our deck and so its population mean is not, in fact, 7. Our “alternative hypothesis” would be that \(\mu \ne 7\). The null hypothesis, that our deck hasn’t been tampered with, would be that \(\mu = 7\). We will reject the null hypothesis if the sample looks sufficiently unlikely to have been produced by the hypotheical null hypothesis population. To quantify “sufficiently unlikely” we choose an “alpha” (\(\alpha\)) value, some low probability. We’re willing to accept that level of risk that we’re making a mistake, rejecting the null hypothesis when we shouldn’t.

The visualization above really is sampling from a fair deck with a mean of 7. But as you’ll see, some of the confidence intervals in the visualization above don’t include the true population mean. They would cause us to reject the null hypothesis (\(\mu = 7\)) and lead us to believe that someone has indeed tampered with the cards. We’d be making a mistake; it just so happened that we obtained a sample for which the confidence interval didn’t contain the true population mean–which is what we can expect to happen with a probability equal to the specified confidence. So in this sense, the interpretation of any single confidence interval is most closely related to the binary decision about the null hypothesis; if the null value lies outside our confidence interval, we reject the null hypothesis; if it’s inside, we don’t reject it.

The deck cards is a convenient example of a known population, but generally speaking we don’t know the true population parameter–that’s the whole reason for taking samples and applying statistics!

jStat = require("https://cdn.jsdelivr.net/npm/jstat@latest/dist/jstat.min.js")

chart = {

  let sampleSize = Number(d3.select("#sampleSize").property("value"));
  let sampling = false;
  let sample = [], sampleArr = [];
  let numberOfCIsThatContainMu = 0;
  const samplingSpeed = 150;

  const w = 1050
  const h = 600
  const margin = {left: 50, right: 50, top: 50, bottom: 50}
  
  let x = d3.scaleLinear()
    .domain([0, 30])
    .range([margin.left, w - margin.right])
  const y = d3.scaleLinear()
    .domain([1, 13])
    .range([h - margin.bottom, margin.top])
    
  const yAxis = d3.axisLeft(y)
    .ticks(11)

  const f = d3.format(".2f")
  const f1 = d3.format(".1f")
  
  const ciInput = document.getElementById('ci-input');
  const sampleSizeInput = d3.select('#sampleSize');
  const buttonRandom = document.getElementById('addRandom');
  

  sampleSizeInput.on("change", () => setup()); 

  let ciWidth = Number(ciInput.value)

  function makeCards(n, divId) {

    d3.select("#" + divId).selectAll("div").remove();
    for (let i = 0; i < n; i++) {

      d3.select("#" + divId).append("div")
        .attr("id", "card" + (i + 1))
        .attr("class", "playing-card-container")
        .classed("playing-card-blank", true)
    }
  }

  function setup() {
    sampleSize = Number(d3.select("#sampleSize").property("value"));
    reset();
    makeCards(sampleSize, "cards");
  }

  function reset() {
    sample = [];
    sampleArr = [];
    numberOfCIsThatContainMu = 0;

    dots.selectAll("circle").remove();
    lines.selectAll("line").remove();

    d3.select("#proportion").text("#");
    d3.select("#count").text("#");
    d3.select("#mean").text("");

    x.domain([0, 30]);
  }

  makeCards(sampleSize, "cards");
  
  
  function addSampleToPlot(values) {
  
    let mean = jStat.mean(values);
    let sd = jStat.stdev(values, true);
    let ci = getCI(values, ciWidth);
    let containsMu = ciContainsMu(mean, ci);
    let n = sampleArr.length + 1;

    numberOfCIsThatContainMu += containsMu;
    
    sampleArr.push({sample: values, mean: mean, ci: ci, containsMu: containsMu, id: n})
    
    drawNewCI(n, mean, ci);

    d3.select("#proportion").text(f1(numberOfCIsThatContainMu / n * 100));
    d3.select("#count").text(n);
    d3.select("#mean").text(f(mean) + " [" + f(mean - ci) + ", " + f(mean + ci) + "]");
  }
  
  buttonRandom.onclick = () => newSample(sampleSize, false);
  d3.select("#drawContinuously").on("click", sampleContinuously);

  function newSample(sampleSize, quick = true) {

    sample = [];
    for (let i = 1; i < sampleSize + 1; i++) {
      const value = Math.floor(Math.random() * 13 + 1);
      sample.push(value);
      d3.select("#card" + i).html(randomCard(value - 1))
    }

    console.log(sample);

    addSampleToPlot(sample);

    if (!quick) {
    d3.selectAll(".playing-card-container")
      .data(sample)
      .classed("playing-card-blank", true)
      .transition().duration(0).delay((d,i) => 100 + i * 100)
      .attr("class", "playing-card-container")
    }

  }
  
  function randomCard(value) {

    const suits = ["\u2660","\u2665","\u2666","\u2663"];
    const colors = ["black", "red", "red", "black"];
    const cards = ["A",2,3,4,5,6,7,8,9,10,"J","Q","K"];
    
    const suitIndex = Math.floor(Math.random() * suits.length);
    const cardIndex = value;

    return "<div class='playing-card-value'>" + cards[cardIndex] + "</div>" + "<div class='playing-card playing-card-" + colors[suitIndex] + "'>" + suits[suitIndex] + "</div>";
  }
  
  ciInput.oninput = function() {
    ciWidth = ciInput.value
    d3.select("#ci-width").text(ciWidth);
    updateCIs(ciWidth)

    const newCI = getCI(sample, ciWidth);
    const m = jStat.mean(sample);
    d3.select("#mean").text(f(m) + " [" + f(m - newCI) + ", " + f(m + newCI) + "]");
  }

let samplingInterval;
const sampleContinuouslyBtn = d3.select("#drawContinuously");

  function sampleContinuously() {

    if (!sampling) {
      sampling = true;
      d3.selectAll(".playing-card-container")
      .classed("playing-card-blank", false);

      samplingInterval = setInterval(() => {
        newSample(sampleSize);
    }, samplingSpeed);

    sampleContinuouslyBtn.html("<i class='bi bi-stop'></i>").attr("class", "btn btn-danger")

    } else {
      sampleContinuouslyBtn.html("<i class='bi bi-play'></i>").attr("class", "btn btn-outline-success")
      clearInterval(samplingInterval);
      sampling = false;
    }
  }

  
  
  const svg = d3.select("#plot-container").append("svg")
    .attr("preserveAspectRatio", "xMinYMin meet")
    .attr("viewBox", "0 0 " + w + " " + h)
    
  const svgDataLayer = svg.append("g")
  const lines = svgDataLayer.append("g")
  const dots = svgDataLayer.append("g")
  const gridY = svg.append("g")
  
  // population mean line
  gridY.append("line")
    .attr("x1", x(0))
    .attr("x2", x(30))
    .attr("y1", y(7))
    .attr("y2", y(7))
    .style("stroke", "grey")
    .attr("stroke-width", 2)
    .attr("stroke-dasharray", [5,5])
  gridY.append("text")
    .attr("x", x(0.2))
    .attr("y", y(7.05))
    .text("true population mean")
    
  
  const axisY = svg.append("g")
  axisY.append("rect").attr("x", -margin.left).attr("width", margin.left).attr("height", h)
    .attr("fill", "var(--mermaid-bg-color)");
  axisY.call(yAxis)
    .attr("transform", `translate(${x(0)},0)`)

function drawNewCI(n, point, margin) {

    const includesMu = ciContainsMu(point, margin);

    dots.append("circle")
      .attr("r", 4)
      .attr("cx", x(n))
      .attr("cy", y(point))

    lines.append("line")
      .attr("x1", x(n))
      .attr("x2", x(n))
      .attr("y1", y(point))
      .attr("y2", y(point))
      .attr("class", "ci-line")
      .classed("ci-contains-mu", includesMu)
      .transition("grow").duration(samplingSpeed)
      .attr("y1", y(point + margin))
      .attr("y2", y(point - margin))
  
    if (n > 30) {
      x.domain([n - 30, n]);

      dots.selectAll("circle").data(sampleArr)
      .transition().duration(samplingSpeed).ease(d3.easeLinear)
      .attr("cx", d => x(d.id));

      lines.selectAll("line").data(sampleArr)
      .transition().duration(samplingSpeed).ease(d3.easeLinear)
      .attr("x1", d => x(d.id)).attr("x2", d => x(d.id));
    } 
  }


  function updateCIs (confidence) {
  
  // take the array and recalculate all CIs
  const n = sampleArr.length;
  numberOfCIsThatContainMu = 0;
  for (var i = 0; i < sampleArr.length; i++) {
    sampleArr[i].ci = getCI(sampleArr[i].sample, confidence);
    sampleArr[i].containsMu = ciContainsMu(sampleArr[i].mean, sampleArr[i].ci);
    numberOfCIsThatContainMu += sampleArr[i].containsMu;
  }
  
  // then redraw all CIs on the svg
  lines.selectAll("line")
  .data(sampleArr)
      .attr("y1", d => y(d.mean + d.ci))
      .attr("y2", d => y(d.mean - d.ci))
      // .attr("class", "ci-line")
      .classed("ci-contains-mu", d => ciContainsMu(d.mean, d.ci))
    
    // and update the description text
    d3.select("#proportion").text(f1(numberOfCIsThatContainMu / n * 100));
  }
  
  
  
}

function getCI (array, confidence) {
  const mean = jStat.mean(array);
  const alpha = 1 - confidence / 100;
  return mean - jStat.tci(mean, alpha, array )[0];

}


function ciContainsMu (point, ci) {
  return (point + ci > 7 && point - ci < 7);
}

function countOfCIs(arr) {
    const x = arr.map(a => a.containsMu);
    const count = x.reduce((acc, curr) => {
  return acc + (curr ? curr : 0);
}, 0);

    return count;
}