From 41c6f038e6328c3520737be7036bc8e906ab36ea Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Sat, 14 Dec 2024 13:55:33 +0100 Subject: [PATCH] Add documenter interlinks and citations (#73) * Remove outdated docs of Distributions integration * Add DocumenterInterLinks as docs dependency * Copy example to `psis` docstring * Link out to MCMCDiagnosticTools docs * Filter out warnings line in doctests * Add MCMCDiagnosticTools as docs dep * Increment patch number * Add references file * Add bibliography to docs * Reference bibliography entry everywhere * Add custom CSS for citations * Match Distributions test dependency bound --- Project.toml | 2 +- docs/Project.toml | 8 ++++- docs/make.jl | 16 ++++++++- docs/src/assets/citations.css | 29 +++++++++++++++ docs/src/index.md | 6 +--- docs/src/references.bib | 24 +++++++++++++ docs/src/references.md | 4 +++ src/core.jl | 66 ++++++++++++++++++++++++++++++----- src/generalized_pareto.jl | 16 ++++----- 9 files changed, 145 insertions(+), 26 deletions(-) create mode 100644 docs/src/assets/citations.css create mode 100644 docs/src/references.bib create mode 100644 docs/src/references.md diff --git a/Project.toml b/Project.toml index fdc6770e..91a56c24 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PSIS" uuid = "ce719bf2-d5d0-4fb9-925d-10a81b42ad04" authors = ["Seth Axen and contributors"] -version = "0.9.6" +version = "0.9.7" [deps] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" diff --git a/docs/Project.toml b/docs/Project.toml index 2f803580..184a9fd7 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,11 +1,17 @@ [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" +DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656" +MCMCDiagnosticTools = "be115224-59cd-429b-ad48-344e309966f0" PSIS = "ce719bf2-d5d0-4fb9-925d-10a81b42ad04" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [compat] -Distributions = "0.23.2, 0.24, 0.25" +Distributions = "0.25.81" Documenter = "1" +DocumenterCitations = "1.2" +DocumenterInterLinks = "1" +MCMCDiagnosticTools = "0.3.2" Plots = "1.10.1" diff --git a/docs/make.jl b/docs/make.jl index 6ac18c64..3366675e 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,21 +1,35 @@ using Distributions, Plots, PSIS using Documenter +using DocumenterCitations +using DocumenterInterLinks DocMeta.setdocmeta!(PSIS, :DocTestSetup, :(using PSIS); recursive=true) +bib = CitationBibliography(joinpath(@__DIR__, "src", "references.bib"); style=:numeric) + +links = InterLinks( + "MCMCDiagnosticTools" => "https://julia.arviz.org/MCMCDiagnosticTools/stable/" +) + makedocs(; modules=[PSIS], authors="Seth Axen and contributors", repo=Remotes.GitHub("arviz-devs", "PSIS.jl"), sitename="PSIS.jl", - format=Documenter.HTML(; prettyurls=get(ENV, "CI", "false") == "true", assets=String[]), + format=Documenter.HTML(; + prettyurls=get(ENV, "CI", "false") == "true", + assets=[joinpath("assets", "citations.css")], + ), pages=[ "Home" => "index.md", "Plotting" => "plotting.md", "API" => "api.md", "Internal" => "internal.md", + "References" => "references.md", ], + doctestfilters=[r"└.*"], # ignore locations in warning messages warnonly=:missing_docs, + plugins=[bib, links], ) deploydocs(; repo="github.com/arviz-devs/PSIS.jl.git", devbranch="main") diff --git a/docs/src/assets/citations.css b/docs/src/assets/citations.css new file mode 100644 index 00000000..db2f1c53 --- /dev/null +++ b/docs/src/assets/citations.css @@ -0,0 +1,29 @@ +/* Adapted from DocumenterCitations.jl's docs */ +.citation dl { + display: grid; + grid-template-columns: max-content auto; +} + +.citation dt { + grid-column-start: 1; +} + +.citation dd { + grid-column-start: 2; + margin-bottom: 0.75em; +} + +.citation ul { + padding: 0 0 2.25em 0; + margin: 0; + list-style: none !important; +} + +.citation ul li { + text-indent: -2.25em; + margin: 0.33em 0.5em 0.5em 2.25em; +} + +.citation ol li { + padding-left: 0.75em; +} \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/index.md index 9028eb7b..5cc5b4f7 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -4,7 +4,7 @@ CurrentModule = PSIS # PSIS -PSIS.jl implements the Pareto smoothed importance sampling (PSIS) algorithm from [^VehtariSimpson2021]. +PSIS.jl implements the Pareto smoothed importance sampling (PSIS) algorithm from [VehtariSimpson2021](@citet). Given a set of importance weights used in some estimator, PSIS both improves the reliability of the estimates by smoothing the importance weights and acts as a diagnostic of the reliability of the estimates. @@ -35,7 +35,3 @@ As indicated by the warnings, this is a poor choice of a proposal distribution, When running PSIS with many parameters, it is useful to plot the Pareto shape values to diagnose convergence. See [Plotting PSIS results](@ref) for examples. - -[^VehtariSimpson2021]: Vehtari A, Simpson D, Gelman A, Yao Y, Gabry J. (2021). - Pareto smoothed importance sampling. - [arXiv:1507.02646v7](https://arxiv.org/abs/1507.02646v7) [stat.CO] diff --git a/docs/src/references.bib b/docs/src/references.bib new file mode 100644 index 00000000..6d39316e --- /dev/null +++ b/docs/src/references.bib @@ -0,0 +1,24 @@ +@article{VehtariSimpson2021, + title = {Pareto Smoothed Importance Sampling}, + author = {Vehtari, Aki and Simpson, Daniel and Gelman, Andrew and Yao, Yuling and Gabry, Jonah}, + year = {2024}, + journal = {Journal of Machine Learning Research}, + volume = {25}, + number = {72}, + pages = {1--58}, + url = {http://jmlr.org/papers/v25/19-556.html}, + eprint = {1507.02646}, + archiveprefix = {arXiv} +} + +@article{ZhangStephens2009, + title = {A New and Efficient Estimation Method for the Generalized Pareto Distribution}, + author = {Zhang, Jin and Stephens, Michael A.}, + year = {2009}, + journal = {Technometrics : a journal of statistics for the physical, chemical, and engineering sciences}, + volume = {51}, + number = {3}, + pages = {316--325}, + publisher = {ASA Website}, + doi = {10.1198/tech.2009.08017}, +} diff --git a/docs/src/references.md b/docs/src/references.md new file mode 100644 index 00000000..78f29bc4 --- /dev/null +++ b/docs/src/references.md @@ -0,0 +1,4 @@ +# References + +```@bibliography +``` diff --git a/src/core.jl b/src/core.jl index 07fe7ccc..7f2cc3d9 100644 --- a/src/core.jl +++ b/src/core.jl @@ -38,7 +38,7 @@ Result of Pareto-smoothed importance sampling (PSIS) using [`psis`](@ref). The `pareto_shape` parameter ``k=ξ`` of the generalized Pareto distribution `tail_dist` can be used to diagnose reliability and convergence of estimates using the importance weights -[^VehtariSimpson2021]. +[VehtariSimpson2021](@citep). - if ``k < \\frac{1}{3}``, importance sampling is stable, and importance sampling (IS) and PSIS both are reliable. @@ -55,9 +55,9 @@ be used to diagnose reliability and convergence of estimates using the importanc See [`PSISPlots.paretoshapeplot`](@ref) for a diagnostic plot. -[^VehtariSimpson2021]: Vehtari A, Simpson D, Gelman A, Yao Y, Gabry J. (2021). - Pareto smoothed importance sampling. - [arXiv:1507.02646v7](https://arxiv.org/abs/1507.02646v7) [stat.CO] +# References + + - [VehtariSimpson2021](@cite) Vehtari et al. JMLR 25:72 (2021). """ struct PSISResult{T,W<:AbstractArray{T},R,L,D} log_weights::W @@ -177,7 +177,7 @@ _pad_right(s, nchars) = "$s" * " "^(nchars - length("$s")) psis(log_ratios, reff = 1.0; kwargs...) -> PSISResult psis!(log_ratios, reff = 1.0; kwargs...) -> PSISResult -Compute Pareto smoothed importance sampling (PSIS) log weights [^VehtariSimpson2021]. +Compute Pareto smoothed importance sampling (PSIS) log weights [VehtariSimpson2021](@citep). While `psis` computes smoothed log weights out-of-place, `psis!` smooths them in-place. @@ -204,9 +204,59 @@ While `psis` computes smoothed log weights out-of-place, `psis!` smooths them in A warning is raised if the Pareto shape parameter ``k ≥ 0.7``. See [`PSISResult`](@ref) for details and [`PSISPlots.paretoshapeplot`](@ref) for a diagnostic plot. -[^VehtariSimpson2021]: Vehtari A, Simpson D, Gelman A, Yao Y, Gabry J. (2021). - Pareto smoothed importance sampling. - [arXiv:1507.02646v7](https://arxiv.org/abs/1507.02646v7) [stat.CO] +# Examples + +Here we smooth log importance ratios for importance sampling 30 isotropic Student +``t``-distributed parameters using standard normal distributions as proposals. + +```jldoctest psis; setup = :(using Random; Random.seed!(42)) +julia> using Distributions + +julia> proposal, target = Normal(), TDist(7); + +julia> x = rand(proposal, 1_000, 1, 30); # (ndraws, nchains, nparams) + +julia> log_ratios = @. logpdf(target, x) - logpdf(proposal, x); + +julia> result = psis(log_ratios) +┌ Warning: 9 parameters had Pareto shape values 0.7 < k ≤ 1. Resulting importance sampling estimates are likely to be unstable. +└ @ PSIS ~/.julia/packages/PSIS/... +┌ Warning: 1 parameters had Pareto shape values k > 1. Corresponding importance sampling estimates are likely to be unstable and are unlikely to converge with additional samples. +└ @ PSIS ~/.julia/packages/PSIS/... +PSISResult with 1000 draws, 1 chains, and 30 parameters +Pareto shape (k) diagnostic values: + Count Min. ESS + (-Inf, 0.5] good 7 (23.3%) 959 + (0.5, 0.7] okay 13 (43.3%) 938 + (0.7, 1] bad 9 (30.0%) —— + (1, Inf) very bad 1 (3.3%) —— +``` + +If the draws were generated using MCMC, we can compute the relative efficiency using +[`MCMCDiagnosticTools.ess`](@extref). + +```jldoctest psis +julia> using MCMCDiagnosticTools + +julia> reff = ess(log_ratios; kind=:basic, split_chains=1, relative=true); + +julia> result = psis(log_ratios, reff) +┌ Warning: 9 parameters had Pareto shape values 0.7 < k ≤ 1. Resulting importance sampling estimates are likely to be unstable. +└ @ PSIS ~/.julia/packages/PSIS/... +┌ Warning: 1 parameters had Pareto shape values k > 1. Corresponding importance sampling estimates are likely to be unstable and are unlikely to converge with additional samples. +└ @ PSIS ~/.julia/packages/PSIS/... +PSISResult with 1000 draws, 1 chains, and 30 parameters +Pareto shape (k) diagnostic values: + Count Min. ESS + (-Inf, 0.5] good 9 (30.0%) 806 + (0.5, 0.7] okay 11 (36.7%) 842 + (0.7, 1] bad 9 (30.0%) —— + (1, Inf) very bad 1 (3.3%) —— +``` + +# References + + - [VehtariSimpson2021](@cite) Vehtari et al. JMLR 25:72 (2021). """ psis, psis! diff --git a/src/generalized_pareto.jl b/src/generalized_pareto.jl index 5d19ca59..38dad13a 100644 --- a/src/generalized_pareto.jl +++ b/src/generalized_pareto.jl @@ -7,9 +7,6 @@ The generalized Pareto distribution. -This is equivalent to `Distributions.GeneralizedPareto` and can be converted to one with -`convert(Distributions.GeneralizedPareto, d)`. - # Constructor GeneralizedPareto(μ, σ, k) @@ -20,8 +17,8 @@ parameter ``σ`` and shape parameter ``k``. !!! note The shape parameter ``k`` is equivalent to the commonly used shape parameter ``ξ``. - This is the same parameterization used by [^VehtariSimpson2021] and is related to that - used by [^ZhangStephens2009] as ``k \\mapsto -k``. + This is the same parameterization used by [VehtariSimpson2021](@citet) and is related to + that used by [ZhangStephens2009](@citet) as ``k \\mapsto -k``. """ struct GeneralizedPareto{T} μ::T @@ -56,7 +53,7 @@ end Fit a [`GeneralizedPareto`](@ref) with location `μ` to the data `x`. -The fit is performed using the Empirical Bayes method of [^ZhangStephens2009]. +The fit is performed using the Empirical Bayes method of [ZhangStephens2009](@citet). # Keywords @@ -67,10 +64,9 @@ The fit is performed using the Empirical Bayes method of [^ZhangStephens2009]. - `min_points::Int=30`: The minimum number of quadrature points to use when estimating the posterior mean of ``\\theta = \\frac{\\xi}{\\sigma}``. -[^ZhangStephens2009]: Jin Zhang & Michael A. Stephens (2009) - A New and Efficient Estimation Method for the Generalized Pareto Distribution, - Technometrics, 51:3, 316-325, - DOI: [10.1198/tech.2009.08017](https://doi.org/10.1198/tech.2009.08017) +# References + + - [ZhangStephens2009](@cite) Zhang & Stephens, Technometrics 51:3 (2009) """ function fit_gpd(x::AbstractArray; prior_adjusted::Bool=true, kwargs...) tail_dist = fit_gpd_empiricalbayes(x; kwargs...)