diff --git a/+nla/+net/+mcc/Base.m b/+nla/+net/+mcc/Base.m index ca4fceea..7cade860 100755 --- a/+nla/+net/+mcc/Base.m +++ b/+nla/+net/+mcc/Base.m @@ -1,6 +1,6 @@ classdef Base methods (Abstract) - p_max = correct(obj, net_atlas, input_struct, prob) + [is_sig_vector, p_max] = correct(obj, net_atlas, input_struct, prob) correction_label = createLabel(obj, net_atlas, input_struct, prob) end end \ No newline at end of file diff --git a/+nla/+net/+mcc/BenjaminiHochberg.m b/+nla/+net/+mcc/BenjaminiHochberg.m index a879373f..23063cad 100755 --- a/+nla/+net/+mcc/BenjaminiHochberg.m +++ b/+nla/+net/+mcc/BenjaminiHochberg.m @@ -4,21 +4,11 @@ end methods - function p_max = correct(obj, net_atlas, input_struct, prob) - [~, p_max] = nla.lib.fdr_bh(prob.v, input_struct.prob_max, 'pdep'); + function [is_sig_vector, p_max] = correct(obj, net_atlas, input_struct, prob) + [is_sig_vector, p_max] = nla.lib.fdr_bh(prob.v, input_struct.prob_max, 'pdep'); end function correction_label = createLabel(obj, net_atlas, input_struct, prob) - p_max = obj.correct(net_atlas, input_struct, prob); - if p_max == 0 - correction_label = sprintf('FDR_{BH} produced no significant nets'); - else - format_specs = "%g/%d tests"; - if isequal(input_struct.behavior_count, 1) - format_specs = "%g/%d test"; - end - correction_label = sprintf(strcat("FDR_{BH}(", format_specs, ")"), input_struct.prob_max * input_struct.behavior_count,... - input_struct.behavior_count); - end + correction_label = sprintf("Benjamini-Hochberg (alpha = %g)",input_struct.prob_max); end end end \ No newline at end of file diff --git a/+nla/+net/+mcc/BenjaminiYekutieli.m b/+nla/+net/+mcc/BenjaminiYekutieli.m index 5c8fb6f9..23e0de8d 100755 --- a/+nla/+net/+mcc/BenjaminiYekutieli.m +++ b/+nla/+net/+mcc/BenjaminiYekutieli.m @@ -4,21 +4,11 @@ end methods - function p_max = correct(obj, net_atlas, input_struct, prob) - [~, p_max] = nla.lib.fdr_bh(prob.v, input_struct.prob_max, 'dep'); + function [is_sig_vector, p_max] = correct(obj, net_atlas, input_struct, prob) + [is_sig_vector, p_max] = nla.lib.fdr_bh(prob.v, input_struct.prob_max, 'dep'); end function correction_label = createLabel(obj, net_atlas, input_struct, prob) - p_max = obj.correct(net_atlas, input_struct, prob); - if p_max == 0 - correction_label = sprintf('FDR_{BY} produced no significant nets'); - else - format_specs = "%g/%d tests"; - if isequal(input_struct.behavior_count, 1) - format_specs = "%g/%d test"; - end - correction_label = sprintf(strcat("FDR_{BY}(", format_specs, ")"), input_struct.prob_max * input_struct.behavior_count,... - input_struct.behavior_count); - end + correction_label = sprintf("Benjamini-Yekutieli (alpha = %g)",input_struct.prob_max); end end end \ No newline at end of file diff --git a/+nla/+net/+mcc/Bonferroni.m b/+nla/+net/+mcc/Bonferroni.m index de306236..8897b413 100755 --- a/+nla/+net/+mcc/Bonferroni.m +++ b/+nla/+net/+mcc/Bonferroni.m @@ -4,15 +4,18 @@ end methods - function p_max = correct(obj, net_atlas, input_struct, prob) + function [is_sig_vector, p_max] = correct(obj, net_atlas, input_struct, prob) + p_max = input_struct.prob_max / net_atlas.numNetPairs(); + is_sig_vector = prob.v < p_max; end function correction_label = createLabel(obj, net_atlas, input_struct, prob) - format_specs_tests = "%d tests"; + format_specs_tests = "%d tests)"; if isequal(input_struct.behavior_count, 1) - format_specs_tests = "%d test"; + format_specs_tests = "%d test)"; end - correction_label = sprintf(strcat("%g/%d net-pairs/", format_specs_tests), input_struct.prob_max * input_struct.behavior_count,... + p_max = input_struct.prob_max / net_atlas.numNetPairs(); + correction_label = sprintf(strcat("P < %.2g (%.2g/%d net-pairs/", format_specs_tests), p_max, input_struct.prob_max * input_struct.behavior_count,... net_atlas.numNetPairs(), input_struct.behavior_count); end end diff --git a/+nla/+net/+mcc/HolmBonferroni.m b/+nla/+net/+mcc/HolmBonferroni.m index 08efeda7..2f8c3995 100644 --- a/+nla/+net/+mcc/HolmBonferroni.m +++ b/+nla/+net/+mcc/HolmBonferroni.m @@ -4,22 +4,16 @@ end methods - function p_max = correct(obj, net_atlas, input_struct, prob) - [is_significant, adjusted_pvals, ~] = nla.lib.bonferroni_holm(prob.v, input_struct.prob_max); - p_max = max(is_significant .* adjusted_pvals); + function [is_sig_vector, p_max] = correct(obj, net_atlas, input_struct, prob) + [is_sig_vector, adjusted_pvals, ~] = nla.lib.bonferroni_holm(prob.v, input_struct.prob_max); + + p_max = max(is_sig_vector .* prob.v); end function correction_label = createLabel(obj, net_atlas, input_struct, prob) - p_max = obj.correct(net_atlas, input_struct, prob); - if p_max == 0 - correction_label = sprintf('Holm-Bonferroni produced no significant networks'); - else - format_specs = "%g/%d tests"; - if isequal(input_struct.behavior_count, 1) - format_specs = "%g/%d test"; - end - correction_label = sprintf(strcat("Holm-Bonferroni(", format_specs, ")"), input_struct.prob_max * input_struct.behavior_count,... - input_struct.behavior_count); - end + correction_label = sprintf("Holm-Bonferroni (alpha = %g)",input_struct.prob_max); + + %Since p threshold is variable with this test, exclude it and + %just use the initial 'alpha' term used in the algorithm. end end end diff --git a/+nla/+net/+mcc/None.m b/+nla/+net/+mcc/None.m index 8ef9dd66..f1bba3df 100755 --- a/+nla/+net/+mcc/None.m +++ b/+nla/+net/+mcc/None.m @@ -4,15 +4,16 @@ end methods - function p_max = correct(obj, net_atlas, input_struct, prob) + function [is_sig_vector, p_max] = correct(obj, net_atlas, input_struct, prob) p_max = input_struct.prob_max; + is_sig_vector = prob.v < p_max; end function correction_label = createLabel(obj, net_atlas, input_struct, prob) - format_specs = "%g/%d tests"; + format_specs = "P < %.2g (%g/%d tests)"; if isequal(input_struct.behavior_count, 1) - format_specs = "%g/%d test"; + format_specs = "P < %.2g (%g/%d test)"; end - correction_label = sprintf(format_specs, input_struct.prob_max * input_struct.behavior_count,... + correction_label = sprintf(format_specs, input_struct.prob_max, input_struct.prob_max * input_struct.behavior_count,... input_struct.behavior_count); end end diff --git a/+nla/+net/+result/+plot/NetworkTestPlot.m b/+nla/+net/+result/+plot/NetworkTestPlot.m index 4e00a3af..b3626707 100644 --- a/+nla/+net/+result/+plot/NetworkTestPlot.m +++ b/+nla/+net/+result/+plot/NetworkTestPlot.m @@ -89,8 +89,8 @@ function getPlotTitle(obj) obj.title = sprintf("%s (D > %g)", obj.title, obj.network_test_options.d_max); end if ~isequal(obj.test_method, "no_permutations") - if isequal(obj.current_settings.ranking, nla.RankingMethod.WINKLER) - obj.title = strcat(obj.title, "\nRanking by Winkler Method"); + if isequal(obj.current_settings.ranking, nla.RankingMethod.FREEDMAN_LANE) + obj.title = strcat(obj.title, "\nRanking by Freedman-Lane Method"); elseif isequal(obj.current_settings.ranking, nla.RankingMethod.WESTFALL_YOUNG) obj.title = strcat(obj.title, "\nRanking by Westfall-Young Method"); else @@ -228,8 +228,8 @@ function resizeFigure(obj, plot_width, plot_height) % All the options (buttons, pulldowns, checkboxes) scale_option = PullDown("plot_scale", "Plot Scale", ["Linear", "Log", "Negative Log10"],... [ProbPlotMethod.DEFAULT, ProbPlotMethod.LOG, ProbPlotMethod.NEGATIVE_LOG_10]); - ranking_method = PullDown("ranking", "Ranking", ["Uncorrected", "Winkler", "Westfall-Young"],... - [RankingMethod.UNCORRECTED, RankingMethod.WINKLER, RankingMethod.WESTFALL_YOUNG]); + ranking_method = PullDown("ranking", "Ranking", ["Uncorrected", "Freedman-Lane", "Westfall-Young"],... + [RankingMethod.UNCORRECTED, RankingMethod.FREEDMAN_LANE, RankingMethod.WESTFALL_YOUNG]); cohens_d = CheckBox("cohens_d", "Cohen's D Threshold", true); centroids = CheckBox("centroids", "ROI Centroids in brain plots", false); multiple_comparison_correction = PullDown("mcc", "Multiple Comparison Correction",... diff --git a/+nla/+net/+result/+plot/NetworkTestPlotApp.mlapp b/+nla/+net/+result/+plot/NetworkTestPlotApp.mlapp index d2513df1..ea13c24c 100644 Binary files a/+nla/+net/+result/+plot/NetworkTestPlotApp.mlapp and b/+nla/+net/+result/+plot/NetworkTestPlotApp.mlapp differ diff --git a/+nla/+net/+result/+plot/NetworkTestPlotApp_exported.m b/+nla/+net/+result/+plot/NetworkTestPlotApp_exported.m index 88dfd8c6..c7fba833 100644 --- a/+nla/+net/+result/+plot/NetworkTestPlotApp_exported.m +++ b/+nla/+net/+result/+plot/NetworkTestPlotApp_exported.m @@ -88,8 +88,8 @@ function getPlotTitle(app) app.title = sprintf("%s (D > %g)", app.title, app.CohensDThresholdEditField.Value); end if ~isequal(app.test_method, "no_permutations") - if isequal(app.RankingDropDown.Value, "nla.RankingMethod.WINKLER") % Look at me, I'm MATLAB. I have no idea why enums are beneficial or how to use them - app.title = strcat(app.title, "\nRanking by Winkler Method"); + if isequal(app.RankingDropDown.Value, "nla.RankingMethod.FREEDMAN_LANE") % Look at me, I'm MATLAB. I have no idea why enums are beneficial or how to use them + app.title = strcat(app.title, "\nRanking by Freedman-Lane Method"); elseif isequal(app.RankingDropDown.Value, "nla.RankingMethod.WESTFALL_YOUNG") app.title = strcat(app.title, "\nRanking by Westfall-Young Method"); else @@ -439,8 +439,8 @@ function createComponents(app) % Create RankingDropDown app.RankingDropDown = uidropdown(app.Panel); - app.RankingDropDown.Items = {'Uncorrected', 'Winkler', 'Westfall-Young'}; - app.RankingDropDown.ItemsData = {'nla.RankingMethod.UNCORRECTED', 'nla.RankingMethod.WINKLER', 'nla.RankingMethod.WESTFALL_YOUNG'}; + app.RankingDropDown.Items = {'Uncorrected', 'Freedman-Lane', 'Westfall-Young'}; + app.RankingDropDown.ItemsData = {'nla.RankingMethod.UNCORRECTED', 'nla.RankingMethod.FREEDMAN_LANE', 'nla.RankingMethod.WESTFALL_YOUNG'}; app.RankingDropDown.ValueChangedFcn = createCallbackFcn(app, @PlotScaleValueChanged, true); app.RankingDropDown.Position = [291 297 100 22]; app.RankingDropDown.Value = 'nla.RankingMethod.UNCORRECTED'; diff --git a/+nla/+net/+result/NetworkResultPlotParameter.m b/+nla/+net/+result/NetworkResultPlotParameter.m index a72e8ad4..c2c81e85 100644 --- a/+nla/+net/+result/NetworkResultPlotParameter.m +++ b/+nla/+net/+result/NetworkResultPlotParameter.m @@ -27,7 +27,7 @@ end function result = plotProbabilityParameters(obj, edge_test_options, edge_test_result, test_method, plot_statistic,... - plot_title, fdr_correction, significance_filter, ranking_method) + plot_title, fdr_correction, effect_size_filter, ranking_method) % plot_title - this will be a string % plot_statistic - this is the stat that will be plotted % significance filter - this will be a boolean or some sort of object (like Cohen's D > D-value) @@ -36,9 +36,9 @@ import nla.TriMatrix nla.TriMatrixDiag % We're going to use a default filter here - if isequal(significance_filter, false) - significance_filter = TriMatrix(obj.number_of_networks, "logical", TriMatrixDiag.KEEP_DIAGONAL); - significance_filter.v = true(numel(significance_filter.v), 1); + if isequal(effect_size_filter, false) + effect_size_filter = TriMatrix(obj.number_of_networks, "logical", TriMatrixDiag.KEEP_DIAGONAL); + effect_size_filter.v = true(numel(effect_size_filter.v), 1); end % Adding on to the plot title if it's a -log10 plot @@ -53,20 +53,26 @@ if isstring(fdr_correction) || ischar(fdr_correction) fdr_correction = nla.net.mcc.(erase(fdr_correction, "-"))(); end - p_value_max = fdr_correction.correct(obj.network_atlas, obj.updated_test_options, statistic_input); + + [is_sig_vector, p_value_max] = fdr_correction.correct(obj.network_atlas, obj.updated_test_options, statistic_input); + p_value_breakdown_label = fdr_correction.createLabel(obj.network_atlas, obj.updated_test_options,... statistic_input); - - name_label = sprintf("%s %s\nP < %.2g (%s)", obj.network_test_results.test_display_name, plot_title,... - p_value_max, p_value_breakdown_label); - if p_value_max == 0 - name_label = sprintf("%s %s\nP = %.2g (%s)", obj.network_test_results.test_display_name, plot_title,... - p_value_max, p_value_breakdown_label); - end + name_label = sprintf("%s %s\n%s", obj.network_test_results.test_display_name, plot_title,... + p_value_breakdown_label); +% else +% +% name_label = sprintf("%s %s\nP < %.2g (%s)", obj.network_test_results.test_display_name, plot_title,... +% p_value_max, p_value_breakdown_label); +% if p_value_max == 0 +% name_label = sprintf("%s %s\nP = %.2g (%s)", obj.network_test_results.test_display_name, plot_title,... +% p_value_max, p_value_breakdown_label); +% end +% end % Filtering if there's a filter provided significance_plot = TriMatrix(obj.number_of_networks, "logical", TriMatrixDiag.KEEP_DIAGONAL); - significance_plot.v = (statistic_input.v < p_value_max) & significance_filter.v; + significance_plot.v = is_sig_vector & effect_size_filter.v; % scale values very slightly for display so numbers just below % the threshold don't show up white but marked significant @@ -195,8 +201,8 @@ function brainFigureButtonCallback(network1, network2) end switch ranking_method - case "nla.RankingMethod.WINKLER" - ranking = "winkler_"; + case "nla.RankingMethod.FREEDMAN_LANE" + ranking = "freedman_lane_"; case "nla.RankingMethod.WESTFALL_YOUNG" ranking = "westfall_young_"; otherwise diff --git a/+nla/+net/+result/NetworkTestResult.m b/+nla/+net/+result/NetworkTestResult.m index e8ff9792..a4068827 100644 --- a/+nla/+net/+result/NetworkTestResult.m +++ b/+nla/+net/+result/NetworkTestResult.m @@ -242,7 +242,7 @@ function createPValueTriMatrices(obj, number_of_networks, test_method) non_correlation_test = any(strcmp(obj.test_name, obj.noncorrelation_input_tests)); uncorrected_names = ["uncorrected_", "legacy_"]; - corrected_names = ["winkler_", "westfall_young_"]; + corrected_names = ["freedman_lane_", "westfall_young_"]; switch test_method case "no_permutations" @@ -285,15 +285,15 @@ function createPValueTriMatrices(obj, number_of_networks, test_method) % I don't really know what these do and haven't really thought about it. Hence the bad naming. function [sig, name] = singleSigMat(obj, network_atlas, edge_test_options, p_value, mcc_method, title_prefix) mcc_method = nla.net.mcc.(mcc_method)(); - p_value_max = mcc_method.correct(network_atlas, edge_test_options, p_value); + [is_sig_vector, p_value_max] = mcc_method.correct(network_atlas, edge_test_options, p_value); p_breakdown_labels = mcc_method.createLabel(network_atlas, edge_test_options, p_value); sig = nla.TriMatrix(network_atlas.numNets(), 'double', nla.TriMatrixDiag.KEEP_DIAGONAL); - sig.v = (p_value.v < p_value_max); - name = sprintf("%s %s P < %.2g (%s)", title_prefix, obj.test_display_name, p_value_max, p_breakdown_labels); - if p_value_max == 0 - name = sprintf("%s %s P = 0 (%s)", title_prefix, obj.test_display_name, p_breakdown_labels); - end + sig.v = is_sig_vector; + name = sprintf("%s %s %s", title_prefix, obj.test_display_name, p_value_max, p_breakdown_labels); +% if p_value_max == 0 +% name = sprintf("%s %s P = 0 (%s)", title_prefix, obj.test_display_name, p_breakdown_labels); +% end end function [number_of_tests, sig_count_mat, names] = appendSignificanceMatrix(... @@ -337,11 +337,11 @@ function createPValueTriMatrices(obj, number_of_networks, test_method) % :return: The full name of the p-value. (example: "single_sammple_p_value") import nla.NetworkLevelMethod - noncorrelation_input_tests = ["chi_squared", "hypergeometric"]; - non_correlation_test = any(strcmp(test_name, noncorrelation_input_tests)); + reference_distribution_tests = ["chi_squared", "hypergeometric"]; + reference_distribution_test = any(strcmp(test_name, reference_distribution_tests)); probability = "two_sample_p_value"; - if isequal(non_correlation_test, false) + if isequal(reference_distribution_test, false) if isequal(test_method, "no_permutations") || isequal(test_method, "within_network_pair") probability = "single_sample_p_value"; end diff --git a/+nla/+net/ResultRank.m b/+nla/+net/ResultRank.m index 4ab3594b..c9cd1788 100644 --- a/+nla/+net/ResultRank.m +++ b/+nla/+net/ResultRank.m @@ -42,8 +42,8 @@ ranking_result = obj.uncorrectedRank(test_method, permutation_results, no_permutation_results, ranking_statistic,... probability, ranking_result); - % Winkler Method ranking - ranking_result = obj.winklerMethodRank(test_method, permutation_results, no_permutation_results, ranking_statistic,... + % Freedman-Lane Method ranking + ranking_result = obj.freedmanLaneMethodRank(test_method, permutation_results, no_permutation_results, ranking_statistic,... probability, ranking_result); % Westfall Young ranking @@ -96,7 +96,7 @@ end - function ranking = winklerMethodRank(obj, test_method, permutation_results, no_permutation_results, ranking_statistic,... + function ranking = freedmanLaneMethodRank(obj, test_method, permutation_results, no_permutation_results, ranking_statistic,... probability, ranking) % Ranks the observed result using method described by Winkler to correct for FWER % @@ -108,7 +108,7 @@ % :param ranking: The NetworkTestResult object to place the results % :return: The same NetworkTestResult object with ranking results - winkler_probability = strcat("winkler_", probability); + freedman_lane_probability = strcat("freedman_lane_", probability); % NET-278: The "max statistic" for hypergeometric is the most significant p-value. Which is the smallest p-value. max_statistic_array = max(abs(permutation_results.(strcat(ranking_statistic, "_permutations")).v)); if isequal(ranking.test_name, "hypergeometric") @@ -117,16 +117,16 @@ for index = 1:numel(no_permutation_results.(strcat("uncorrected_", probability)).v) if isequal(ranking.test_name, "hypergeometric") - ranking.(test_method).(winkler_probability).v(index) = sum(... + ranking.(test_method).(freedman_lane_probability).v(index) = sum(... squeeze(max_statistic_array) <= abs(no_permutation_results.(ranking_statistic).v(index))... ); else - ranking.(test_method).(winkler_probability).v(index) = sum(... + ranking.(test_method).(freedman_lane_probability).v(index) = sum(... squeeze(max_statistic_array) >= abs(no_permutation_results.(ranking_statistic).v(index))... ); end end - ranking.(test_method).(winkler_probability).v = ranking.(test_method).(winkler_probability).v ./ obj.permutations; + ranking.(test_method).(freedman_lane_probability).v = ranking.(test_method).(freedman_lane_probability).v ./ obj.permutations; end function ranking = westfallYoungMethodRank(obj, test_method, permutation_results, no_permutation_results, ranking_statistic,... diff --git a/+nla/+net/unittests/ResultRankTestCase.m b/+nla/+net/unittests/ResultRankTestCase.m index 0f65c4dd..8f75ffe4 100644 --- a/+nla/+net/unittests/ResultRankTestCase.m +++ b/+nla/+net/unittests/ResultRankTestCase.m @@ -114,11 +114,11 @@ function fullConnectomeUncorrectedRankingTest(testCase) testCase.verifyEqual(uncorrected_results, expected_uncorrected); end - function fullConnectomeWinklerRankingTest(testCase) - winkler_results = testCase.rank.full_connectome.winkler_two_sample_p_value.v; - expected_winkler = testCase.rank_results.full_connectome.winkler_two_sample_p_value.v; + function fullConnectomeFreedmanLaneRankingTest(testCase) + freedman_lane_results = testCase.rank.full_connectome.freedman_lane_two_sample_p_value.v; + expected_freedman_lane = testCase.rank_results.full_connectome.freedman_lane_two_sample_p_value.v; - testCase.verifyEqual(winkler_results, expected_winkler); + testCase.verifyEqual(freedman_lane_results, expected_freedman_lane); end function fullConnectomeWestfallYoungRankingTest(testCase) @@ -140,11 +140,11 @@ function withinNetworkPairUncorrectedRankingTest(testCase) testCase.verifyEqual(uncorrected_results, expected_uncorrected); end - function withinNetworkPairWinklerRankingTest(testCase) - winkler_results = testCase.rank.within_network_pair.winkler_single_sample_p_value.v; - expected_winkler = testCase.rank_results.within_network_pair.winkler_single_sample_p_value.v; + function withinNetworkPairFreedmanLaneRankingTest(testCase) + freedman_lane_results = testCase.rank.within_network_pair.freedman_lane_single_sample_p_value.v; + expected_freedman_lane = testCase.rank_results.within_network_pair.freedman_lane_single_sample_p_value.v; - testCase.verifyEqual(winkler_results, expected_winkler); + testCase.verifyEqual(freedman_lane_results, expected_freedman_lane); end function withinNetworkPairWestfallYoungRankingTest(testCase) diff --git a/+nla/RankingMethod.m b/+nla/RankingMethod.m index cb3f98b1..978f90ac 100644 --- a/+nla/RankingMethod.m +++ b/+nla/RankingMethod.m @@ -1,5 +1,5 @@ classdef RankingMethod enumeration - UNCORRECTED, WINKLER, WESTFALL_YOUNG + UNCORRECTED, FREEDMAN_LANE, WESTFALL_YOUNG end end \ No newline at end of file diff --git a/NLA_GUI.mlapp b/NLA_GUI.mlapp index 36853e50..fd6e8f7e 100755 Binary files a/NLA_GUI.mlapp and b/NLA_GUI.mlapp differ diff --git a/docs/source/methodology.rst b/docs/source/methodology.rst index 2ea5220d..28026972 100644 --- a/docs/source/methodology.rst +++ b/docs/source/methodology.rst @@ -45,10 +45,6 @@ Benjamini-Hochberg, Westfall and Young :cite:p:`WestfallP`, and Freedman-Lane :c functional connectivity data are not shuffled in order to preserve the inherent covariant structure of the data across permutations -.. note:: - Freedman-Lane FWER correction is referred to as Winkler in NLA. Implementation was modeled after the algorithm described in the paper - by Winkler. - Brain Network Map Selection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/network_level_results.rst b/docs/source/network_level_results.rst index edc91855..17c43fc7 100644 --- a/docs/source/network_level_results.rst +++ b/docs/source/network_level_results.rst @@ -49,7 +49,7 @@ number of points above or below (depending on the test used) the non-permuted (o In NLA this is referred to as "ranking" since it is simply counting values in a sorted list. This basic ranking is referred to as the "uncorrected" *p*-value. There are two other options for ranking in NLA. These account for :abbr:`FWER (family-wise error rate)`. The first method is based off the "randomise" method :cite:p:`FreedmanD,WinklerA`. -This is referred to as the "Winkler method". The second method is called "Westfall-Young" in NLA described by +The second method is called "Westfall-Young" in NLA described by an alogrithm :cite:p:`WestfallP` by Westfall and Young. Result Rank @@ -66,7 +66,7 @@ Result Rank .. mat:automethod:: uncorrectedRank - .. mat:automethod:: winklerMethodRank + .. mat:automethod:: freedmanLaneMethodRank .. mat:automethod:: westfallYoungMethodRank