diff --git a/monai/metrics/confusion_matrix.py b/monai/metrics/confusion_matrix.py index c471200069..deb4d8b693 100644 --- a/monai/metrics/confusion_matrix.py +++ b/monai/metrics/confusion_matrix.py @@ -43,12 +43,18 @@ class ConfusionMatrixMetric(CumulativeIterationMetric): ``"miss rate"``, ``"fall out"``, ``"false discovery rate"``, ``"false omission rate"``, ``"prevalence threshold"``, ``"threat score"``, ``"accuracy"``, ``"balanced accuracy"``, ``"f1 score"``, ``"matthews correlation coefficient"``, ``"fowlkes mallows index"``, - ``"informedness"``, ``"markedness"``] + ``"informedness"``, ``"markedness"``, ``"positive likelihood ratio"``, + ``"negative likelihood ratio"``, ``"diagnostic odds ratio"``] Some of the metrics have multiple aliases (as shown in the wikipedia page aforementioned), and you can also input those names instead. Except for input only one metric, multiple metrics are also supported via input a sequence of metric names, such as ("sensitivity", "precision", "recall"), if ``compute_sample`` is ``True``, multiple ``f`` and ``not_nans`` will be returned with the same order as input names when calling the class. + Note that for the likelihood-ratio family — ``"positive likelihood ratio"`` (``LR+ = tpr / fpr``), + ``"negative likelihood ratio"`` (``LR- = fnr / tnr``) and ``"diagnostic odds ratio"`` + (``DOR = (tp * tn) / (fp * fn)``) — a zero denominator yields ``NaN`` (e.g. ``LR+`` is undefined + when the false positive rate is ``0``, and ``DOR`` when either ``fp`` or ``fn`` is ``0``), + following the same convention as the other ratio-based metrics. compute_sample: when reducing, if ``True``, each sample's metric will be computed based on each confusion matrix first. if ``False``, compute reduction on the confusion matrices first, defaults to ``False``. reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values, @@ -226,7 +232,8 @@ def compute_confusion_matrix_metric(metric_name: str, confusion_matrix: torch.Te ``"miss rate"``, ``"fall out"``, ``"false discovery rate"``, ``"false omission rate"``, ``"prevalence threshold"``, ``"threat score"``, ``"accuracy"``, ``"balanced accuracy"``, ``"f1 score"``, ``"matthews correlation coefficient"``, ``"fowlkes mallows index"``, - ``"informedness"``, ``"markedness"``] + ``"informedness"``, ``"markedness"``, ``"positive likelihood ratio"``, + ``"negative likelihood ratio"``, ``"diagnostic odds ratio"``] Some of the metrics have multiple aliases (as shown in the wikipedia page aforementioned), and you can also input those names instead. confusion_matrix: Please see the doc string of the function ``get_confusion_matrix`` for more details. @@ -304,6 +311,19 @@ def compute_confusion_matrix_metric(metric_name: str, confusion_matrix: torch.Te npv = torch.where((tn + fn) > 0, tn / (tn + fn), nan_tensor) numerator = ppv + npv - 1.0 denominator = 1.0 + elif metric == "plr": + # LR+ = sensitivity / (1 - specificity) = tpr / fpr; fpr == 0 yields NaN + tpr = torch.where(p > 0, tp / p, nan_tensor) + fpr = torch.where(n > 0, fp / n, nan_tensor) + numerator, denominator = tpr, fpr + elif metric == "nlr": + # LR- = (1 - sensitivity) / specificity = fnr / tnr; tnr == 0 yields NaN + fnr = torch.where(p > 0, fn / p, nan_tensor) + tnr = torch.where(n > 0, tn / n, nan_tensor) + numerator, denominator = fnr, tnr + elif metric == "dor": + # DOR = (LR+) / (LR-) = (tp * tn) / (fp * fn); fp * fn == 0 yields NaN + numerator, denominator = tp * tn, fp * fn else: raise NotImplementedError("the metric is not implemented.") @@ -360,4 +380,10 @@ def check_confusion_matrix_metric_name(metric_name: str) -> str: return "bm" if metric_name in ["markedness", "deltap", "mk"]: return "mk" + if metric_name in ["positive_likelihood_ratio", "plr", "lr+"]: + return "plr" + if metric_name in ["negative_likelihood_ratio", "nlr", "lr-"]: + return "nlr" + if metric_name in ["diagnostic_odds_ratio", "dor"]: + return "dor" raise NotImplementedError("the metric is not implemented.") diff --git a/tests/metrics/test_compute_confusion_matrix.py b/tests/metrics/test_compute_confusion_matrix.py index 2486049d79..e8b3cb6937 100644 --- a/tests/metrics/test_compute_confusion_matrix.py +++ b/tests/metrics/test_compute_confusion_matrix.py @@ -107,6 +107,9 @@ torch.tensor([0.5000, 0.6036, 0.3536]), torch.tensor([0.3333, 0.2500, 0.1667]), torch.tensor([0.3333, 0.3333, 0.0833]), + torch.tensor([0.0000, 1.0000, 1.5000]), + torch.tensor([0.7500, 0.7500, 0.7500]), + torch.tensor([0.0000, 1.0000, 0.0000]), ] result_mean = [ torch.tensor([0.5000]), @@ -126,6 +129,9 @@ torch.tensor([0.4857]), torch.tensor([0.2500]), torch.tensor([0.2500]), + torch.tensor([1.6667]), + torch.tensor([0.7500]), + torch.tensor([0.3333]), ] metric_names = [ "tpr", @@ -145,6 +151,9 @@ "fm", "bm", "mk", + "plr", + "nlr", + "dor", ] result: Any = None for idx, item in enumerate(metric_names): @@ -218,6 +227,92 @@ torch.tensor([[[0.0, 0.0, 46137344.0, 0.0]]]), ] +# 5. likelihood ratios and diagnostic odds ratio: hand-computed values, including undefined cases +# sample 0: tp=80, fp=20, tn=70, fn=20 -> tpr=0.8, fpr=2/9 -> LR+=3.6; fnr=0.2, tnr=7/9 -> LR-=0.25714...; +# DOR=(80*70)/(20*20)=14 +# sample 1: tp=10, fp=0, tn=90, fn=0 -> fpr=0 -> LR+ is NaN; fnr=0 -> LR-=0.0; fp*fn=0 -> DOR is NaN +# sample 2: tp=5, fp=5, tn=0, fn=0 -> tpr=1, fpr=1 -> LR+=1; tnr=0 -> LR- is NaN; fp*fn=0 -> DOR is NaN +# sample 3: tp=0, fp=10, tn=10, fn=10 -> tpr=0, fpr=0.5 -> LR+=0; fnr=1, tnr=0.5 -> LR-=2; DOR=(0*10)/(10*10)=0 +# sample 4: tp=0, fp=0, tn=0, fn=0 -> empty class -> LR+ / LR- / DOR are all NaN +# (row order is [tp, fp, tn, fn], matching compute_confusion_matrix output) +TEST_CASE_LR = [ + torch.tensor( + [ + [80.0, 20.0, 70.0, 20.0], + [10.0, 0.0, 90.0, 0.0], + [5.0, 5.0, 0.0, 0.0], + [0.0, 10.0, 10.0, 10.0], + [0.0, 0.0, 0.0, 0.0], + ] + ) +] + +# classification-style input, channel-wise hand-computed values, no undefined cases: +# ch0: tp=2, fp=1, tn=1, fn=1 -> LR+=4/3, LR-=2/3, DOR=2; ch1: tp=1, fp=1, tn=2, fn=1 -> LR+=3/2, LR-=3/4, DOR=2 +TEST_CASE_LR_CLF = [ + { + "y_pred": torch.tensor([[1, 0], [1, 0], [0, 1], [1, 0], [0, 1]]), + "y": torch.tensor([[1, 0], [1, 0], [0, 1], [0, 1], [1, 0]]), + "include_background": True, + "metric_name": ["positive likelihood ratio", "negative likelihood ratio", "diagnostic odds ratio"], + "reduction": "sum_batch", + }, + [torch.tensor([4.0 / 3.0, 3.0 / 2.0]), torch.tensor([2.0 / 3.0, 3.0 / 4.0]), torch.tensor([2.0, 2.0])], +] + +# same input with the background channel excluded: only ch1 values remain +TEST_CASE_LR_CLF_NOBG = [ + { + "y_pred": torch.tensor([[1, 0], [1, 0], [0, 1], [1, 0], [0, 1]]), + "y": torch.tensor([[1, 0], [1, 0], [0, 1], [0, 1], [1, 0]]), + "include_background": False, + "metric_name": ["lr+", "lr-", "dor"], + "reduction": "sum_batch", + }, + [torch.tensor([3.0 / 2.0]), torch.tensor([3.0 / 4.0]), torch.tensor([2.0])], +] + +# 6. likelihood ratios / DOR on the multi-class data with undefined entries, exercising all-reduction +# modes. From the per-sample confusion matrices in the data_nan comment above +# ([tp, fp, tn, fn] per channel): +# plr: [[0, 1, 0], [nan, nan, nan], [nan, nan, nan]] +# nlr: [[1.5, 1.0, 1.5], [nan, nan, nan], [1.0, 1.0, nan]] +# dor: [[0, 1, 0], [nan, nan, nan], [nan, nan, nan]] +# so with NaN-ignoring reductions: plr sum=1 (3 not-nans) / sum_batch=[0,1,0] ([1,1,1] not-nans); +# nlr sum=6 (5 not-nans) / sum_batch=[2.5,2.0,1.5] ([2,2,1] not-nans); dor same as plr. +TEST_CASES_LR_NAN = [] +_metric_names = ["positive likelihood ratio", "negative likelihood ratio", "diagnostic odds ratio"] +_result_sum = [torch.tensor([1.0]), torch.tensor([6.0]), torch.tensor([1.0])] +_notnans_sum = [torch.tensor([3.0]), torch.tensor([5.0]), torch.tensor([3.0])] +_result_sum_batch = [torch.tensor([0.0, 1.0, 0.0]), torch.tensor([2.5, 2.0, 1.5]), torch.tensor([0.0, 1.0, 0.0])] +_notnans_sum_batch = [torch.tensor([1.0, 1.0, 1.0]), torch.tensor([2.0, 2.0, 1.0]), torch.tensor([1.0, 1.0, 1.0])] +# mean_channel reduces within each sample (over channels): plr/dor keep only sample 0 (1/3), +# nlr keeps sample 0 at 4/3 and sample 2 at 1.0; all-NaN samples reduce to 0 with 0 not-nans. +_result_mean_channel = [ + torch.tensor([1.0 / 3.0, 0.0, 0.0]), + torch.tensor([4.0 / 3.0, 0.0, 1.0]), + torch.tensor([1.0 / 3.0, 0.0, 0.0]), +] +_notnans_mean_channel = [torch.tensor([3.0, 0.0, 0.0]), torch.tensor([3.0, 0.0, 2.0]), torch.tensor([3.0, 0.0, 0.0])] +for _idx in range(3): + for _reduction in ["sum", "sum_batch", "mean_channel"]: + _test_case = [data_nan.copy()] + _test_case[0]["compute_sample"] = True + _test_case[0]["include_background"] = True + _test_case[0]["metric_name"] = _metric_names[_idx] + _test_case[0]["reduction"] = _reduction + _test_case[0]["get_not_nans"] = True + if _reduction == "sum": + _test_case.append(_result_sum[_idx]) + _test_case.append(_notnans_sum[_idx]) + elif _reduction == "sum_batch": + _test_case.append(_result_sum_batch[_idx]) + _test_case.append(_notnans_sum_batch[_idx]) + else: + _test_case.append(_result_mean_channel[_idx]) + _test_case.append(_notnans_mean_channel[_idx]) + TEST_CASES_LR_NAN.append(_test_case) + class TestConfusionMatrix(unittest.TestCase): @parameterized.expand([TEST_CASE_CONFUSION_MATRIX]) @@ -289,6 +384,102 @@ def test_precision(self, input_data, expected_value): assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4) np.testing.assert_equal(result.device, input_data["y_pred"].device) + @parameterized.expand([TEST_CASE_LR]) + def test_likelihood_ratios(self, confusion_matrix): + """Check likelihood-ratio / DOR aliases and edge cases on a per-class confusion matrix. + + Args: + confusion_matrix: a stacked [5, 4] confusion-matrix tensor with manually specified + counts, each row in ``[tp, fp, tn, fn]`` order. + """ + # every advertised spelling must resolve to the same result (case/space-insensitive) + for alias in ("lr+", "plr", "Positive Likelihood Ratio", "POSITIVE_LIKELIHOOD_RATIO"): + plr = compute_confusion_matrix_metric(alias, confusion_matrix) + assert_allclose(plr[0], torch.tensor(3.6), atol=1e-4, rtol=1e-4) + self.assertTrue(torch.isnan(plr[1])) + assert_allclose(plr[2], torch.tensor(1.0), atol=1e-4, rtol=1e-4) + assert_allclose(plr[3], torch.tensor(0.0), atol=1e-4, rtol=1e-4) + self.assertTrue(torch.isnan(plr[4])) + for alias in ("lr-", "nlr", "Negative Likelihood Ratio", "NEGATIVE_LIKELIHOOD_RATIO"): + nlr = compute_confusion_matrix_metric(alias, confusion_matrix) + assert_allclose(nlr[0], torch.tensor(0.2 / (70.0 / 90.0)), atol=1e-4, rtol=1e-4) + assert_allclose(nlr[1], torch.tensor(0.0), atol=1e-4, rtol=1e-4) + # sample 2 has tn == 0 (zero specificity), so LR- is undefined -> NaN + self.assertTrue(torch.isnan(nlr[2])) + assert_allclose(nlr[3], torch.tensor(2.0), atol=1e-4, rtol=1e-4) + self.assertTrue(torch.isnan(nlr[4])) + for alias in ("dor", "Diagnostic Odds Ratio", "DIAGNOSTIC_ODDS_RATIO"): + dor = compute_confusion_matrix_metric(alias, confusion_matrix) + assert_allclose(dor[0], torch.tensor(14.0), atol=1e-4, rtol=1e-4) + # samples 1 / 2 have fp * fn == 0, so the odds ratio is undefined -> NaN + self.assertTrue(torch.isnan(dor[1])) + self.assertTrue(torch.isnan(dor[2])) + assert_allclose(dor[3], torch.tensor(0.0), atol=1e-4, rtol=1e-4) + self.assertTrue(torch.isnan(dor[4])) + + @parameterized.expand([TEST_CASE_LR]) + def test_dor_equals_lr_ratio(self, confusion_matrix): + """DOR must equal LR+ / LR- wherever all three are defined (algebraic consistency). + + Args: + confusion_matrix: a stacked [5, 4] confusion-matrix tensor with manually specified + counts, each row in ``[tp, fp, tn, fn]`` order. + """ + plr = compute_confusion_matrix_metric("plr", confusion_matrix) + nlr = compute_confusion_matrix_metric("nlr", confusion_matrix) + dor = compute_confusion_matrix_metric("dor", confusion_matrix) + defined = ~torch.isnan(plr) & ~torch.isnan(nlr) & ~torch.isnan(dor) & (nlr != 0) + self.assertGreater(defined.sum().item(), 0) + assert_allclose(dor[defined], (plr[defined] / nlr[defined]), atol=1e-4, rtol=1e-4) + # wherever DOR is NaN at least one of LR+ / LR- must be undefined or zero-denominator too + self.assertTrue(torch.isnan(dor[~defined]).all()) + + @parameterized.expand([TEST_CASE_LR_CLF, TEST_CASE_LR_CLF_NOBG]) + def test_likelihood_ratios_clf(self, input_data, expected_values): + """Check likelihood ratios / DOR through the ``ConfusionMatrixMetric`` classification API. + + Args: + input_data: keyword arguments for ``ConfusionMatrixMetric`` plus ``y_pred``/``y`` + to feed the metric. + expected_values: expected per-channel LR+ / LR- / DOR values after aggregation. + """ + params = input_data.copy() + vals = {} + vals["y_pred"] = params.pop("y_pred") + vals["y"] = params.pop("y") + metric = ConfusionMatrixMetric(**params) + metric(**vals) + results = metric.aggregate() + # one aggregated channel per requested metric, in the same order as expected_values + self.assertEqual(len(results), len(expected_values)) + for result, expected_value in zip(results, expected_values): + assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4) + + @parameterized.expand(TEST_CASES_LR_NAN) + def test_likelihood_reductions_with_nan(self, input_data, expected_value, expected_not_nans): + """Check LR+ / LR- / DOR under NaN-ignoring reductions with undefined entries present. + + Args: + input_data: keyword arguments for ``ConfusionMatrixMetric`` plus ``y_pred``/``y`` + to feed the metric (the multi-class ``data_nan`` inputs). + expected_value: expected reduced metric value. + expected_not_nans: expected count of defined entries feeding the reduction. + """ + params = input_data.copy() + vals = {} + vals["y_pred"] = params.pop("y_pred") + vals["y"] = params.pop("y") + metric = ConfusionMatrixMetric(**params) + metric(**vals) + result, not_nans = metric.aggregate()[0] + assert_allclose(result, expected_value, atol=1e-4, rtol=1e-4) + assert_allclose(not_nans, expected_not_nans, atol=1e-4, rtol=1e-4) + + def test_unknown_metric_name(self): + """An unimplemented metric name must raise ``NotImplementedError``.""" + with self.assertRaises(NotImplementedError): + compute_confusion_matrix_metric("diagnostic likelihood ratio", TEST_CASE_LR[0]) + if __name__ == "__main__": unittest.main()