diff --git a/Project.toml b/Project.toml index 177f8a5..e863cba 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TuringCallbacks" uuid = "ea0860ee-d0ef-45ef-82e6-cc37d6be2f9c" authors = ["Tor Erlend Fjelde and contributors"] -version = "0.3.1" +version = "0.4.0" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" @@ -26,8 +26,8 @@ DocStringExtensions = "0.8, 0.9" OnlineStats = "1.5" Reexport = "0.2, 1.0" Requires = "1" -TensorBoardLogger = "0.1" -Turing = "0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.20, 0.21, 0.22" +TensorBoardLogger = "0.1.22" +Turing = "0.29" julia = "1" [extras] diff --git a/ext/TuringCallbacksTuringExt.jl b/ext/TuringCallbacksTuringExt.jl index b18fad7..9eb9d16 100644 --- a/ext/TuringCallbacksTuringExt.jl +++ b/ext/TuringCallbacksTuringExt.jl @@ -1,26 +1,69 @@ module TuringCallbacksTuringExt if isdefined(Base, :get_extension) - using Turing: Turing + using Turing: Turing, DynamicPPL using TuringCallbacks: TuringCallbacks else # Requires compatible. - using ..Turing: Turing + using ..Turing: Turing, DynamicPPL using ..TuringCallbacks: TuringCallbacks end -const TuringTransition = Union{Turing.Inference.Transition,Turing.Inference.HMCTransition} +const TuringTransition = Union{ + Turing.Inference.Transition, + Turing.Inference.SMCTransition, + Turing.Inference.PGTransition +} -function TuringCallbacks.params_and_values(transition::TuringTransition; kwargs...) - return Iterators.map(zip(Turing.Inference._params_to_array([transition])...)) do (ksym, val) - return string(ksym), val - end +function TuringCallbacks.params_and_values( + model::DynamicPPL.Model, + transition::TuringTransition; + kwargs... +) + vns, vals = Turing.Inference._params_to_array(model, [transition]) + return zip(Iterators.map(string, vns), vals) end -function TuringCallbacks.extras(transition::TuringTransition; kwargs...) - return Iterators.map(zip(Turing.Inference.get_transition_extras([transition])...)) do (ksym, val) - return string(ksym), val - end +function TuringCallbacks.extras( + model::DynamicPPL.Model, transition::TuringTransition; + kwargs... +) + names, vals = Turing.Inference.get_transition_extras([transition]) + return zip(string.(names), vec(vals)) +end + +default_hyperparams(sampler::DynamicPPL.Sampler) = default_hyperparams(sampler.alg) +default_hyperparams(alg::Turing.Inference.InferenceAlgorithm) = ( + string(f) => getfield(alg, f) for f in fieldnames(typeof(alg)) +) + +const AlgsWithDefaultHyperparams = Union{ + Turing.Inference.HMC, + Turing.Inference.HMCDA, + Turing.Inference.NUTS, + Turing.Inference.SGHMC, + +} + +function TuringCallbacks.hyperparams( + model::DynamicPPL.Model, + sampler::DynamicPPL.Sampler{<:AlgsWithDefaultHyperparams}; + kwargs... +) + return default_hyperparams(sampler) +end + +function TuringCallbacks.hyperparam_metrics( + model, + sampler::Turing.Sampler{<:Turing.Inference.NUTS} +) + return [ + "extras/acceptance_rate/stat/Mean", + "extras/max_hamiltonian_energy_error/stat/Mean", + "extras/lp/stat/Mean", + "extras/n_steps/stat/Mean", + "extras/tree_depth/stat/Mean" + ] end end diff --git a/src/TuringCallbacks.jl b/src/TuringCallbacks.jl index 758a2e4..1d183c3 100644 --- a/src/TuringCallbacks.jl +++ b/src/TuringCallbacks.jl @@ -19,6 +19,7 @@ end export DefaultDict, WindowStat, Thin, Skip, TensorBoardCallback, MultiCallback +include("utils.jl") include("stats.jl") include("tensorboardlogger.jl") include("callbacks/tensorboard.jl") diff --git a/src/callbacks/save.jl b/src/callbacks/save.jl new file mode 100644 index 0000000..638f78f --- /dev/null +++ b/src/callbacks/save.jl @@ -0,0 +1,37 @@ +############################### +### Saves samples on the go ### +############################### + +""" + SaveCSV + +A callback saves samples to .csv file during sampling +""" +function SaveCSV( + rng::AbstractRNG, + model::Model, + sampler::Sampler, + transition, + state, + iteration::Int64; + kwargs..., +) + SaveCSV(rng, model, sampler, transition, state.vi, iteration; kwargs...) +end + +function SaveCSV( + rng::AbstractRNG, + model::Model, + sampler::Sampler, + transition, + vi::AbstractVarInfo, + iteration::Int64; + kwargs..., +) + vii = deepcopy(vi) + invlink!!(vii, model) + θ = vii[sampler] + # it would be good to have the param names as in the chain + chain_name = get(kwargs, :chain_name, "chain") + write(string(chain_name, ".csv"), Dict("params" => [θ]); append = true, delim = ";") +end diff --git a/src/callbacks/tensorboard.jl b/src/callbacks/tensorboard.jl index 9d28f9a..0f1dc1d 100644 --- a/src/callbacks/tensorboard.jl +++ b/src/callbacks/tensorboard.jl @@ -30,9 +30,22 @@ provided instead of `lg`. particular variable and value; expected signature is `filter(varname, value)`. If `isnothing` a default-filter constructed from `exclude` and `include` will be used. -- `exclude = nothing`: If non-empty, these variables will not be logged. -- `include = nothing`: If non-empty, only these variables will be logged. +- `exclude = String[]`: If non-empty, these variables will not be logged. +- `include = String[]`: If non-empty, only these variables will be logged. - `include_extras::Bool = true`: Include extra statistics from transitions. +- `extras_include = String[]`: If non-empty, only these extra statistics will be logged. +- `extras_exclude = String[]`: If non-empty, these extra statistics will not be logged. +- `extras_filter = nothing`: Filter determining whether or not we should log + extra statistics; expected signature is `filter(extra, value)`. + If `isnothing` a default-filter constructed from `extras_exclude` and + `extras_include` will be used. +- `include_hyperparams::Bool = true`: Include hyperparameters. +- `hyperparam_include = String[]`: If non-empty, only these hyperparameters will be logged. +- `hyperparam_exclude = String[]`: If non-empty, these hyperparameters will not be logged. +- `hyperparam_filter = nothing`: Filter determining whether or not we should log + hyperparameters; expected signature is `filter(hyperparam, value)`. + If `isnothing` a default-filter constructed from `hyperparam_exclude` and + `hyperparam_include` will be used. - `directory::String = nothing`: if specified, will together with `comment` be used to define the logging directory. - `comment::String = nothing`: if specified, will together with `directory` be used to @@ -41,19 +54,21 @@ provided instead of `lg`. # Fields $(TYPEDFIELDS) """ -struct TensorBoardCallback{L,F,VI,VE} +struct TensorBoardCallback{L,F1,F2,F3} "Underlying logger." logger::AbstractLogger "Lookup for variable name to statistic estimate." stats::L - "Filter determining whether or not we should log stats for a particular variable." - filter::F - "Variables to include in the logging." - include::VI - "Variables to exclude from the logging." - exclude::VE + "Filter determining whether to include stats for a particular variable." + variable_filter::F1 "Include extra statistics from transitions." include_extras::Bool + "Filter determining whether to include a particular extra statistic." + extras_filter::F2 + "Include hyperparameters." + include_hyperparams::Bool + "Filter determining whether to include a particular hyperparameter." + hyperparam_filter::F3 "Prefix used for logging realizations/parameters" param_prefix::String "Prefix used for logging extra statistics" @@ -77,18 +92,37 @@ function TensorBoardCallback(args...; comment = "", directory = nothing, kwargs. return TensorBoardCallback(lg, args...; kwargs...) end +maybe_filter(f; kwargs...) = f +maybe_filter(::Nothing; exclude=nothing, include=nothing) = NameFilter(; exclude, include) + function TensorBoardCallback( lg::AbstractLogger, stats = nothing; num_bins::Int = 100, exclude = nothing, include = nothing, - include_extras::Bool = true, filter = nothing, + include_extras::Bool = true, + extras_include = nothing, + extras_exclude = nothing, + extras_filter = nothing, + include_hyperparams::Bool = false, + hyperparams_include = nothing, + hyperparams_exclude = nothing, + hyperparams_filter = nothing, param_prefix::String = "", extras_prefix::String = "extras/", kwargs... ) + # Create the filters. + variable_filter_f = maybe_filter(filter; include=include, exclude=exclude) + extras_filter_f = maybe_filter( + extras_filter; include=extras_include, exclude=extras_exclude + ) + hyperparams_filter_f = maybe_filter( + hyperparams_filter; include=hyperparams_include, exclude=hyperparams_exclude + ) + # Lookups: create default ones if not given stats_lookup = if stats isa OnlineStat # Warn the user if they've provided a non-empty `OnlineStat` @@ -107,7 +141,15 @@ function TensorBoardCallback( end return TensorBoardCallback( - lg, stats_lookup, filter, include, exclude, include_extras, param_prefix, extras_prefix + lg, + stats_lookup, + variable_filter_f, + include_extras, + extras_filter_f, + include_hyperparams, + hyperparams_filter_f, + param_prefix, + extras_prefix ) end @@ -117,23 +159,11 @@ end Filter parameters and values from a `transition` based on the `filter` of `cb`. """ function filter_param_and_value(cb::TensorBoardCallback, param, value) - if !isnothing(cb.filter) - return cb.filter(param, value) - end - - # Otherwise we construct from `include` and `exclude`. - if !isnothing(cb.include) - # If only `include` is given, we only return the variables in `include`. - return param ∈ cb.include - elseif !isnothing(cb.exclude) - # If only `exclude` is given, we return all variables except those in `exclude`. - return !(param ∈ cb.exclude) - end - - # Otherwise we return `true` by default. - return true + return cb.variable_filter(param, value) +end +function filter_param_and_value(cb::TensorBoardCallback, param_and_value::Tuple) + filter_param_and_value(cb, param_and_value...) end -filter_param_and_value(cb::TensorBoardCallback, param_and_value::Tuple) = filter_param_and_value(cb, param_and_value...) """ default_param_names_for_values(x) @@ -144,25 +174,84 @@ default_param_names_for_values(x) = ("θ[$i]" for i = 1:length(x)) """ - params_and_values(transition[, state]; kwargs...) + params_and_values(model, transition[, state]; kwargs...) params_and_values(model, sampler, transition, state; kwargs...) Return an iterator over parameter names and values from a `transition`. """ -params_and_values(transition, state; kwargs...) = params_and_values(transition; kwargs...) -params_and_values(model, sampler, transition, state; kwargs...) = params_and_values(transition, state; kwargs...) +function params_and_values(model, transition, state; kwargs...) + return params_and_values(model, transition; kwargs...) +end +function params_and_values(model, sampler, transition, state; kwargs...) + return params_and_values(model, transition, state; kwargs...) +end """ - extras(transition[, state]; kwargs...) + extras(model, transition[, state]; kwargs...) extras(model, sampler, transition, state; kwargs...) Return an iterator with elements of the form `(name, value)` for additional statistics in `transition`. Default implementation returns an empty iterator. """ -extras(transition; kwargs...) = () -extras(transition, state; kwargs...) = extras(transition; kwargs...) -extras(model, sampler, transition, state; kwargs...) = extras(transition, state; kwargs...) +extras(model, transition; kwargs...) = () +extras(model, transition, state; kwargs...) = extras(model, transition; kwargs...) +function extras(model, sampler, transition, state; kwargs...) + return extras(model, transition, state; kwargs...) +end + +""" + filter_extras_and_value(cb::TensorBoardCallback, name, value) + +Filter extras and values from a `transition` based on the `filter` of `cb`. +""" +function filter_extras_and_value(cb::TensorBoardCallback, name, value) + return cb.extras_filter(name, value) +end +function filter_extras_and_value(cb::TensorBoardCallback, name_and_value::Tuple) + return filter_extras_and_value(cb, name_and_value...) +end + +""" + hyperparams(model, sampler[, transition, state]; kwargs...) + +Return an iterator with elements of the form `(name, value)` for hyperparameters in `model`. +""" +function hyperparams(model, sampler; kwargs...) + @warn "`hyperparams(model, sampler; kwargs...)` is not implemented for $(typeof(model)) and $(typeof(sampler)). If you want to record hyperparameters, please implement this method." + return Pair{String, Any}[] +end +function hyperparams(model, sampler, transition, state; kwargs...) + return hyperparams(model, sampler; kwargs...) +end + +""" + filter_hyperparams_and_value(cb::TensorBoardCallback, name, value) + +Filter hyperparameters and values from a `transition` based on the `filter` of `cb`. +""" +function filter_hyperparams_and_value(cb::TensorBoardCallback, name, value) + return cb.hyperparam_filter(name, value) +end +function filter_hyperparams_and_value( + cb::TensorBoardCallback, + name_and_value::Union{Pair,Tuple} +) + return filter_hyperparams_and_value(cb, name_and_value...) +end + +""" + hyperparam_metrics(model, sampler[, transition, state]; kwargs...) + +Return a `Vector{String}` of metrics for hyperparameters in `model`. +""" +function hyperparam_metrics(model, sampler; kwargs...) + @warn "`hyperparam_metrics(model, sampler; kwargs...)` is not implemented for $(typeof(model)) and $(typeof(sampler)). If you want to use some of the other recorded values as hyperparameters metrics, please implement this method." + return String[] +end +function hyperparam_metrics(model, sampler, transition, state; kwargs...) + return hyperparam_metrics(model, sampler; kwargs...) +end increment_step!(lg::TensorBoardLogger.TBLogger, Δ_Step) = TensorBoardLogger.increment_step!(lg, Δ_Step) @@ -170,11 +259,32 @@ increment_step!(lg::TensorBoardLogger.TBLogger, Δ_Step) = function (cb::TensorBoardCallback)(rng, model, sampler, transition, state, iteration; kwargs...) stats = cb.stats lg = cb.logger - filterf = Base.Fix1(filter_param_and_value, cb) + variable_filter = Base.Fix1(filter_param_and_value, cb) + extras_filter = Base.Fix1(filter_extras_and_value, cb) + hyperparams_filter = Base.Fix1(filter_hyperparams_and_value, cb) + + if iteration == 1 && cb.include_hyperparams + # If it's the first iteration, we write the hyperparameters. + hparams = Dict(Iterators.filter( + hyperparams_filter, + hyperparams(model, sampler, transition, state; kwargs...) + )) + if !isempty(hparams) + TensorBoardLogger.write_hparams!( + lg, + hparams, + hyperparam_metrics(model, sampler) + ) + end + end + # TODO: Should we use the explicit interface for TensorBoardLogger? with_logger(lg) do - for (k, val) in Iterators.filter(filterf, params_and_values(transition, state; kwargs...)) + for (k, val) in Iterators.filter( + variable_filter, + params_and_values(model, sampler, transition, state; kwargs...) + ) stat = stats[k] # Log the raw value @@ -189,8 +299,18 @@ function (cb::TensorBoardCallback)(rng, model, sampler, transition, state, itera # Transition statstics if cb.include_extras - for (name, val) in extras(transition, state; kwargs...) + for (name, val) in Iterators.filter( + extras_filter, + extras(model, sampler, transition, state; kwargs...) + ) @info "$(cb.extras_prefix)$(name)" val + + # TODO: Make this customizable. + if val isa Real + stat = stats["$(cb.extras_prefix)$(name)"] + fit!(stat, float(val)) + @info ("$(cb.extras_prefix)$(name)") stat + end end end # Increment the step for the logger. diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 0000000..2220659 --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,10 @@ +Base.@kwdef struct NameFilter{A,B} + include::A=nothing + exclude::B=nothing +end + +(f::NameFilter)(name, value) = f(name) +function (f::NameFilter)(name) + include, exclude = f.include, f.exclude + (exclude === nothing || name ∉ exclude) && (include === nothing || name ∈ include) +end diff --git a/test/multicallback.jl b/test/multicallback.jl new file mode 100644 index 0000000..69ba91f --- /dev/null +++ b/test/multicallback.jl @@ -0,0 +1,21 @@ +@testset "MultiCallback" begin + # Number of MCMC samples/steps + num_samples = 100 + num_adapts = 50 + + # Sampling algorithm to use + alg = NUTS(num_adapts, 0.65) + + callback = MultiCallback(CountingCallback(), CountingCallback()) + chain = sample(demo_model, alg, num_samples, callback=callback) + + # Both should have been trigger an equal number of times. + counts = map(c -> c.count[], callback.callbacks) + @test counts[1] == counts[2] + @test counts[1] == num_samples + + # Add a new one and make sure it's not like the others. + callback = TuringCallbacks.push!!(callback, CountingCallback()) + counts = map(c -> c.count[], callback.callbacks) + @test counts[1] == counts[2] != counts[3] +end diff --git a/test/runtests.jl b/test/runtests.jl index 78d8225..cdb8aa6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,56 +9,21 @@ end (c::CountingCallback)(args...; kwargs...) = c.count[] += 1 -@testset "TuringCallbacks.jl" begin - # TODO: Improve. - @model function demo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, √s) - for i in eachindex(x) - x[i] ~ Normal(m, √s) - end - end - - xs = randn(100) .+ 1 - model = demo(xs) - - # Number of MCMC samples/steps - num_samples = 1_000 - num_adapts = 500 - - # Sampling algorithm to use - alg = NUTS(num_adapts, 0.65) - - @testset "MultiCallback" begin - callback = MultiCallback(CountingCallback(), CountingCallback()) - chain = sample(model, alg, num_samples, callback=callback) - - # Both should have been trigger an equal number of times. - counts = map(c -> c.count[], callback.callbacks) - @test counts[1] == counts[2] - @test counts[1] == num_samples - - # Add a new one and make sure it's not like the others. - callback = TuringCallbacks.push!!(callback, CountingCallback()) - counts = map(c -> c.count[], callback.callbacks) - @test counts[1] == counts[2] != counts[3] +@model function demo(x) + s ~ InverseGamma(2, 3) + m ~ Normal(0, √s) + for i in eachindex(x) + x[i] ~ Normal(m, √s) end +end - @testset "TensorBoardCallback" begin - # Create the callback - callback = TensorBoardCallback(mktempdir()) - - # Sample - chain = sample(model, alg, num_samples; callback=callback) - - # Extract the values. - hist = convert(MVHistory, callback.logger) +function DynamicPPL.TestUtils.varnames(::DynamicPPL.Model{typeof(demo)}) + return [@varname(s), @varname(m)] +end - # Compare the recorded values to the chain. - m_mean = last(last(hist["m/stat/Mean"])) - s_mean = last(last(hist["s/stat/Mean"])) +const demo_model = demo(randn(100) .+ 1) - @test m_mean ≈ mean(chain[:m]) - @test s_mean ≈ mean(chain[:s]) - end +@testset "TuringCallbacks.jl" begin + include("multicallback.jl") + include("tensorboardcallback.jl") end diff --git a/test/save.jl b/test/save.jl new file mode 100644 index 0000000..c29dc95 --- /dev/null +++ b/test/save.jl @@ -0,0 +1,7 @@ +@testset "SaveCallback" begin + # Sample + sample(model, alg, num_samples; callback = SaveCSV, chain_name="chain_1") + chain = Matrix(CSV.read("chain_1.csv", DataFrame, header=false)) + @test size(chain) == (num_samples, 2) + rm("chain_1.csv") +end \ No newline at end of file diff --git a/test/tensorboardcallback.jl b/test/tensorboardcallback.jl new file mode 100644 index 0000000..a8043d7 --- /dev/null +++ b/test/tensorboardcallback.jl @@ -0,0 +1,163 @@ +@testset "TensorBoardCallback" begin + tmpdir = mktempdir() + mkpath(tmpdir) + + vns = DynamicPPL.TestUtils.varnames(demo_model) + + # Number of MCMC samples/steps + num_samples = 100 + num_adapts = 50 + + # Sampling algorithm to use + alg = NUTS(num_adapts, 0.65) + + @testset "Correctness of values" begin + # Create the callback + callback = TensorBoardCallback(joinpath(tmpdir, "runs")) + + # Sample + chain = sample(demo_model, alg, num_samples; callback=callback) + + # Extract the values. + hist = convert(MVHistory, callback.logger) + + # Compare the recorded values to the chain. + m_mean = last(last(hist["m/stat/Mean"])) + s_mean = last(last(hist["s/stat/Mean"])) + + @test m_mean ≈ mean(chain[:m]) + @test s_mean ≈ mean(chain[:s]) + end + + @testset "Default" begin + # Create the callback + callback = TensorBoardCallback( + joinpath(tmpdir, "runs"); + ) + + # Sample + chain = sample(demo_model, alg, num_samples; callback=callback) + + # Read the logging info. + hist = convert(MVHistory, callback.logger) + + # Check the variables. + @testset "$vn" for vn in vns + # Should have the `val` field. + @test haskey(hist, Symbol(vn, "/val")) + # Should have the `Mean` and `Variance` stat. + @test haskey(hist, Symbol(vn, "/stat/Mean")) + @test haskey(hist, Symbol(vn, "/stat/Variance")) + end + + # Check the extra statistics. + @testset "extras" begin + @test haskey(hist, Symbol("extras/lp/val")) + @test haskey(hist, Symbol("extras/acceptance_rate/val")) + end + end + + @testset "Exclude variable" begin + # Create the callback + callback = TensorBoardCallback( + joinpath(tmpdir, "runs"); + exclude=["s"] + ) + + # Sample + chain = sample(demo_model, alg, num_samples; callback=callback) + + # Read the logging info. + hist = convert(MVHistory, callback.logger) + + # Check the variables. + @testset "$vn" for vn in vns + if vn == @varname(s) + @test !haskey(hist, Symbol(vn, "/val")) + @test !haskey(hist, Symbol(vn, "/stat/Mean")) + @test !haskey(hist, Symbol(vn, "/stat/Variance")) + else + @test haskey(hist, Symbol(vn, "/val")) + @test haskey(hist, Symbol(vn, "/stat/Mean")) + @test haskey(hist, Symbol(vn, "/stat/Variance")) + end + end + + # Check the extra statistics. + @testset "extras" begin + @test haskey(hist, Symbol("extras/lp/val")) + @test haskey(hist, Symbol("extras/acceptance_rate/val")) + end + end + + @testset "Exclude extras" begin + # Create the callback + callback = TensorBoardCallback( + joinpath(tmpdir, "runs"); + include_extras=false + ) + + # Sample + chain = sample(demo_model, alg, num_samples; callback=callback) + + # Read the logging info. + hist = convert(MVHistory, callback.logger) + + # Check the variables. + @testset "$vn" for vn in vns + @test haskey(hist, Symbol(vn, "/val")) + @test haskey(hist, Symbol(vn, "/stat/Mean")) + @test haskey(hist, Symbol(vn, "/stat/Variance")) + end + + # Check the extra statistics. + @testset "extras" begin + @test !haskey(hist, Symbol("extras/lp/val")) + @test !haskey(hist, Symbol("extras/acceptance_rate/val")) + end + end + + @testset "With hyperparams" begin + @testset "$alg (has hyperparam: $hashyp)" for (alg, hashyp) in [ + (HMC(0.05, 10), true), + (HMCDA(num_adapts, 0.65, 1.0), true), + (NUTS(num_adapts, 0.65), true), + (MH(), false), + ] + + # Create the callback + callback = TensorBoardCallback( + joinpath(tmpdir, "runs"); + include_hyperparams=true, + ) + + # Sample + chain = sample(demo_model, alg, num_samples; callback=callback) + + # HACK: This touches internals so might just break at some point. + # If it some point does, let's just remove this test. + # Inspiration: https://github.com/JuliaLogging/TensorBoardLogger.jl/blob/3d9c1a554a08179785459ad7b83bce0177b90275/src/Deserialization/deserialization.jl#L244-L258 + iter = TensorBoardLogger.TBEventFileCollectionIterator( + callback.logger.logdir, purge=true + ) + + found_one = false + for event_file in iter + for event in event_file + event.what === nothing && continue + !(event.what.value isa TensorBoardLogger.Summary) && continue + + for (tag, _) in event.what.value + if tag == "_hparams_/experiment" + found_one = true + break + end + end + end + + found_one && break + end + @test (hashyp && found_one) || (!hashyp && !found_one) + end + end +end