Pairwise method comparison

Difference metric explanation

Rather than comparing absolute method performance (see effect sizes), here we compare PGS effect sizes β relative to each other. For this, a baseline PGS against which we compare (baseline method) has to be chosen.

For each PGS with effect size βₓ, we calculate the difference to the baseline method with effect size βᵧ. Confidence intervals are calculated adjusting for the correlation between PGS x and y. These confidence intervals form the basis of significance tests reported in the manuscript.

The “raw” differences (βₓ - βᵧ) are on the same scale as the original effect sizes. Relative differences ((βₓ - βᵧ) / βᵧ) are often reported in methods comparisons, as different phenotypes and target data may have different baseline performance.

This interactive plot corresponds to Figure 1C in the published article.

viewof method_y = Inputs.radio(
  ["dbslmm", "sbayesr", "lassosum", "prscs", "ldpred2", "megaprs", "pt.clump", "UKBB.EnsPRS"],
  { value: ["dbslmm", "sbayesr", "lassosum", "prscs", "ldpred2", "megaprs", "pt.clump", "UKBB.EnsPRS"],
  value: "pt.clump",
  label: "Baseline method:",
  sort: true, 
  unique: true,
  disabled: disable_method_type[method_type]}
)

viewof method_type = Inputs.radio(["CV", "auto"],
{ label: "Tuning method:",
  value: "CV"})
  
viewof endpoint = Inputs.select(endpoints, {value: "T2D", label: "Endpoint:"})

viewof use_relative = Inputs.radio(["relative", "raw"],
{ label: "Difference metric",
  value: "relative"})

Plot.plot({
  grid: true,
  marginLeft: 150,
  color: {
    domain: ["dbslmm", "sbayesr", "lassosum", "prscs", "ldpred2", "megaprs", "pt.clump", "UKBB.EnsPRS"],
    range: ["#5778a4", "#e49444", "#d1615d", "#85b6b2", "#6a9f58", "#e7ca60", "#a87c9f", "#f1a2a9"]
  },
  symbol: {
    legend: true
  },
  style: {
    fontSize: "12px"
  },
  facet: {
    data: eur, 
    y: "bbid",
    marginRight: 75,
  },
  fy: {
    label: "Biobank"
  },
  x: {
    label: x_map[use_relative].get("x_label")
  },
  y: {
    label: "Score development method"
  },
  marks: [
    Plot.frame(),
    Plot.link(eur, {
      x1: x_map[use_relative].get("error_bar_low"),
      x2: x_map[use_relative].get("error_bar_high"),
      y1: "method_x",
      y2: "method_x"
    }),
    Plot.text([`EUR`], {
      fy: ["bbid"],
      frameAnchor: "bottom-right",
      lineWidth: 18,
      dx: -5,
      dy: -5
    }),
    Plot.ruleX([0], {stroke: "lightgray", strokeWidth: 2, strokeDasharray: "4 2"}),
    Plot.dot(eur, {
      x: x_map[use_relative].get("x"),
      y: "method_x",
      fill: "method_x",
      symbol: "method_type_x"
    })
  ]
})

Plot.plot({
  grid: true,
  marginLeft: 150,
  color: {
    domain: ["dbslmm", "sbayesr", "lassosum", "prscs", "ldpred2", "megaprs", "pt.clump", "UKBB.EnsPRS"],
    range: ["#5778a4", "#e49444", "#d1615d", "#85b6b2", "#6a9f58", "#e7ca60", "#a87c9f", "#f1a2a9"]
  },
  symbol: {
    legend: false
  },
  style: {
    fontSize: "12px"
  },
  facet: {
    data: sas, 
    y: "bbid",
    marginRight: 75,
    marginTop: 50
  },
  fy: {
    label: "Biobank"
  },
  x: {
    label: x_map[use_relative].get("x_label")
  },
  y: {
    label: "Score development method"
  },
  marks: [
    Plot.frame(),
    Plot.dot(sas, {
      x: x_map[use_relative].get("x"),
      y: "method_x",
      fill: "method_x",
      symbol: "method_type_x"
    }),
    Plot.link(sas, {
      x1: x_map[use_relative].get("error_bar_low"),
      x2: x_map[use_relative].get("error_bar_high"),
      y1: "method_x",
      y2: "method_x"
    }),
    Plot.text([`SAS`], {
      fy: ["bbid"],
      frameAnchor: "bottom-right",
      lineWidth: 18,
      dx: -5,
      dy: -5
    }),
    Plot.ruleX([0], {stroke: "lightgray", strokeWidth: 2, strokeDasharray: "4 2"})
  ]
})

beta_map = new Map([["x_label", "βₓ - βᵧ"], ["x", "beta_diff"], ["error_bar_low", "ci_low_diff"], ["error_bar_high", "ci_high_diff"]])
relative_map = new Map([["x_label", "(βₓ - βᵧ) / βᵧ"], ["x", "relative_beta_diff"], ["error_bar_low", "relative_ci_low_diff"], ["error_bar_high", "relative_ci_high_diff"]])
x_map = ({ "relative": relative_map, "raw": beta_map })

// some methods only have one method type, so disable the radio button
disable_method_type = ({ "CV": ["prscs", "sbayesr", "dbslmm"], "auto": ["UKBB.EnsPRS"]})

import { aq, op } from '@uwdata/arquero'

filtered = aq.fromJSON(dst)
  .select("method_x", "bbid", "ancestry", "method_x", "method_type_x", "beta_diff", "beta_y", "ci_low_diff", "ci_high_diff", "phenotype", "method_y", "method_type_y")
  .derive({ relative_beta_diff: d => d.beta_diff / d.beta_y,
    relative_ci_high_diff: d => d.ci_high_diff / d.beta_y,
    relative_ci_low_diff: d => d.ci_low_diff / d.beta_y })
  .params({endpoint: endpoint, method_y: method_y, method_type: method_type})
  .filter((d, $) => op.includes($.method_type, d.method_type_y ) &&
    op.includes($.method_y, d.method_y) &&
    op.includes($.endpoint, d.phenotype ))
  
// need to filter before plotting to prevent empty facets appearing
eur = filtered.filter(d => d.ancestry == "EUR")
sas = filtered.filter(d => d.ancestry == "SAS")

Conclusion

If you’re using pt.clump (without cross validation) to develop new polygenic risk scores for type 2 diabetes, you could make a better score by using any of the other tested methods.