From 4f591afc71f9ec509bfc82103a6889715b8ab28a Mon Sep 17 00:00:00 2001 From: zuheng Date: Mon, 10 Jul 2023 23:28:55 -0700 Subject: [PATCH 01/40] add support for hasconverged --- src/train.jl | 14 ++++++++++++-- test/interface.jl | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/train.jl b/src/train.jl index 8682795b..76165429 100644 --- a/src/train.jl +++ b/src/train.jl @@ -144,6 +144,8 @@ Iteratively updating the parameters `θ` of the normalizing flow `re(θ)` by cal which returns a dictionary-like object of statistics to be displayed in the progress bar. re and θ are used for reconstructing the normalizing flow in case that user want to further axamine the status of the flow. +- `hasconverged = (iter, opt_stats, re, θ, st) -> false`: function that checks whether the + training has converged. The default is to always return false. - `prog=ProgressMeter.Progress( max_iters; desc="Training", barlen=31, showspeed=true, enabled=show_progress )`: progress bar configuration @@ -164,6 +166,7 @@ function optimize( optimiser::Optimisers.AbstractRule=Optimisers.ADAM(), show_progress::Bool=true, callback=nothing, + hasconverged=(i, stats, re, θ, st) -> false, prog=ProgressMeter.Progress( max_iters; desc="Training", barlen=31, showspeed=true, enabled=show_progress ), @@ -176,10 +179,13 @@ function optimize( st = Optimisers.setup(optimiser, θ) # TODO: Add support for general `hasconverged(...)` approach to allow early termination. - time_elapsed = @elapsed for i in 1:max_iters + converged = false + i = 0 + time_elapsed = @elapsed while (i ≤ max_iters) && !converged + # Compute gradient and objective value; results are stored in `diff_results` grad!(rng, ad, vo, θ, re, diff_result, args...) - # save stats + # Save stats ls = DiffResults.value(diff_result) g = DiffResults.gradient(diff_result) stat = (iteration=i, loss=ls, gradient_norm=norm(g)) @@ -193,6 +199,10 @@ function optimize( # update optimiser state and parameters st, θ = Optimisers.update!(st, θ, DiffResults.gradient(diff_result)) + + # check convergence + converged = hasconverged(i, opt_stats, re, θ, st) + i += 1 pm_next!(prog, stat) end diff --git a/test/interface.jl b/test/interface.jl index ed80609b..14a47bc7 100644 --- a/test/interface.jl +++ b/test/interface.jl @@ -19,6 +19,8 @@ ) sample_per_iter = 10 + cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) + checkconv(iter, opt_stats, re, θ, st) = iter > 4999 flow_trained, stats, _ = train_flow( elbo, flow, @@ -28,6 +30,8 @@ optimiser=Optimisers.ADAM(0.01 * one(T)), ADbackend=adtype, show_progress=false, + callback=cb, + hasconverged=checkconv, ) θ, re = Optimisers.destructure(flow_trained) From 4e31807e491942be89b2db00e88597e9eb1463e5 Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 09:36:49 -0700 Subject: [PATCH 02/40] fix test error --- src/train.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/train.jl b/src/train.jl index 76165429..a8e07a45 100644 --- a/src/train.jl +++ b/src/train.jl @@ -180,7 +180,7 @@ function optimize( # TODO: Add support for general `hasconverged(...)` approach to allow early termination. converged = false - i = 0 + i = 1 time_elapsed = @elapsed while (i ≤ max_iters) && !converged # Compute gradient and objective value; results are stored in `diff_results` grad!(rng, ad, vo, θ, re, diff_result, args...) From 99ac0c1fc4c6698cfe5a5f9be0cb45f75d786098 Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 10:28:43 -0700 Subject: [PATCH 03/40] rm example/Manifest.toml --- example/Manifest.toml | 1713 ----------------------------------------- 1 file changed, 1713 deletions(-) delete mode 100644 example/Manifest.toml diff --git a/example/Manifest.toml b/example/Manifest.toml deleted file mode 100644 index 5d370aa8..00000000 --- a/example/Manifest.toml +++ /dev/null @@ -1,1713 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.9.1" -manifest_format = "2.0" -project_hash = "f2ecc355a33269e2baf01e4a296e89edd7540dbc" - -[[deps.ADTypes]] -git-tree-sha1 = "e58c18d2312749847a74f5be80bb0fa53da102bd" -uuid = "47edcb42-4c32-4615-8424-f2b9edc5f35b" -version = "0.1.5" - -[[deps.AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "8bc0aaec0ca548eb6cf5f0d7d16351650c1ee956" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.3.2" -weakdeps = ["ChainRulesCore"] - - [deps.AbstractFFTs.extensions] - AbstractFFTsChainRulesCoreExt = "ChainRulesCore" - -[[deps.Adapt]] -deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" -uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "3.6.2" -weakdeps = ["StaticArrays"] - - [deps.Adapt.extensions] - AdaptStaticArraysExt = "StaticArrays" - -[[deps.ArgCheck]] -git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" -uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" -version = "2.3.0" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.1" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[deps.Atomix]] -deps = ["UnsafeAtomics"] -git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" -uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" -version = "0.1.0" - -[[deps.BFloat16s]] -deps = ["LinearAlgebra", "Printf", "Random", "Test"] -git-tree-sha1 = "dbf84058d0a8cbbadee18d25cf606934b22d7c66" -uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" -version = "0.4.2" - -[[deps.BangBang]] -deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] -git-tree-sha1 = "e28912ce94077686443433c2800104b061a827ed" -uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" -version = "0.3.39" - - [deps.BangBang.extensions] - BangBangChainRulesCoreExt = "ChainRulesCore" - BangBangDataFramesExt = "DataFrames" - BangBangStaticArraysExt = "StaticArrays" - BangBangStructArraysExt = "StructArrays" - BangBangTypedTablesExt = "TypedTables" - - [deps.BangBang.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" - TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[deps.Baselet]] -git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" -uuid = "9718e550-a3fa-408a-8086-8db961cd8217" -version = "0.1.1" - -[[deps.BenchmarkTools]] -deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] -git-tree-sha1 = "d9a9701b899b30332bbcb3e1679c41cce81fb0e8" -uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -version = "1.3.2" - -[[deps.Bijectors]] -deps = ["ArgCheck", "ChainRulesCore", "ChangesOfVariables", "Compat", "Distributions", "Functors", "InverseFunctions", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "MappedArrays", "Random", "Reexport", "Requires", "Roots", "SparseArrays", "Statistics"] -git-tree-sha1 = "6a7c5ac8d300e59131b3875f6e6e86248810943b" -uuid = "76274a88-744f-5084-9051-94815aaf08c4" -version = "0.13.1" - - [deps.Bijectors.extensions] - BijectorsDistributionsADExt = "DistributionsAD" - BijectorsForwardDiffExt = "ForwardDiff" - BijectorsLazyArraysExt = "LazyArrays" - BijectorsReverseDiffExt = "ReverseDiff" - BijectorsTrackerExt = "Tracker" - BijectorsZygoteExt = "Zygote" - - [deps.Bijectors.weakdeps] - DistributionsAD = "ced4e74d-a319-5a8a-b0ac-84af2272839c" - ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" - LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" - ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" - Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" - Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" - -[[deps.BitFlags]] -git-tree-sha1 = "43b1a4a8f797c1cddadf60499a8a077d4af2cd2d" -uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" -version = "0.1.7" - -[[deps.Bzip2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2" -uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" -version = "1.0.8+0" - -[[deps.CEnum]] -git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" -uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" -version = "0.4.2" - -[[deps.CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "280893f920654ebfaaaa1999fbd975689051f890" -uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "4.2.0" - -[[deps.CUDA_Driver_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] -git-tree-sha1 = "498f45593f6ddc0adff64a9310bb6710e851781b" -uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" -version = "0.5.0+1" - -[[deps.CUDA_Runtime_Discovery]] -deps = ["Libdl"] -git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" -uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" -version = "0.2.2" - -[[deps.CUDA_Runtime_jll]] -deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "5248d9c45712e51e27ba9b30eebec65658c6ce29" -uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -version = "0.6.0+0" - -[[deps.CUDNN_jll]] -deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "2918fbffb50e3b7a0b9127617587afa76d4276e8" -uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" -version = "8.8.1+0" - -[[deps.Cairo_jll]] -deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2" -uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" -version = "1.16.1+1" - -[[deps.Calculus]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" -uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" -version = "0.5.1" - -[[deps.ChainRules]] -deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] -git-tree-sha1 = "61549d9b52c88df34d21bd306dba1d43bb039c87" -uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "1.51.0" - -[[deps.ChainRulesCore]] -deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644" -uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.16.0" - -[[deps.ChangesOfVariables]] -deps = ["LinearAlgebra", "Test"] -git-tree-sha1 = "f84967c4497e0e1955f9a582c232b02847c5f589" -uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" -version = "0.1.7" - -[[deps.CodeTracking]] -deps = ["InteractiveUtils", "UUIDs"] -git-tree-sha1 = "d730914ef30a06732bdd9f763f6cc32e92ffbff1" -uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" -version = "1.3.1" - -[[deps.CodecZlib]] -deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "9c209fb7536406834aa938fb149964b985de6c83" -uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.1" - -[[deps.ColorSchemes]] -deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] -git-tree-sha1 = "be6ab11021cd29f0344d5c4357b163af05a48cba" -uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" -version = "3.21.0" - -[[deps.ColorTypes]] -deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "eb7f0f8307f71fac7c606984ea5fb2817275d6e4" -uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.11.4" - -[[deps.ColorVectorSpace]] -deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "SpecialFunctions", "Statistics", "TensorCore"] -git-tree-sha1 = "600cc5508d66b78aae350f7accdb58763ac18589" -uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" -version = "0.9.10" - -[[deps.Colors]] -deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] -git-tree-sha1 = "fc08e5930ee9a4e03f84bfb5211cb54e7769758a" -uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.10" - -[[deps.CommonSolve]] -git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c" -uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" -version = "0.2.4" - -[[deps.CommonSubexpressions]] -deps = ["MacroTools", "Test"] -git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" -uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" -version = "0.3.0" - -[[deps.Compat]] -deps = ["UUIDs"] -git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.7.0" -weakdeps = ["Dates", "LinearAlgebra"] - - [deps.Compat.extensions] - CompatLinearAlgebraExt = "LinearAlgebra" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.2+0" - -[[deps.CompositionsBase]] -git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" -uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" -version = "0.1.2" -weakdeps = ["InverseFunctions"] - - [deps.CompositionsBase.extensions] - CompositionsBaseInverseFunctionsExt = "InverseFunctions" - -[[deps.ConcurrentUtilities]] -deps = ["Serialization", "Sockets"] -git-tree-sha1 = "96d823b94ba8d187a6d8f0826e731195a74b90e9" -uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.2.0" - -[[deps.ConstructionBase]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "738fec4d684a9a6ee9598a8bfee305b26831f28c" -uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.2" - - [deps.ConstructionBase.extensions] - ConstructionBaseIntervalSetsExt = "IntervalSets" - ConstructionBaseStaticArraysExt = "StaticArrays" - - [deps.ConstructionBase.weakdeps] - IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - -[[deps.ContextVariablesX]] -deps = ["Compat", "Logging", "UUIDs"] -git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" -uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" -version = "0.1.3" - -[[deps.Contour]] -git-tree-sha1 = "d05d9e7b7aedff4e5b51a029dced05cfb6125781" -uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" -version = "0.6.2" - -[[deps.DataAPI]] -git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.15.0" - -[[deps.DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "d1fff3a548102f48987a52a2e0d114fa97d730f0" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.13" - -[[deps.DataValueInterfaces]] -git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" -uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" -version = "1.0.0" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[deps.DefineSingletons]] -git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" -uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" -version = "0.1.2" - -[[deps.DelimitedFiles]] -deps = ["Mmap"] -git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" -uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" -version = "1.9.1" - -[[deps.DiffResults]] -deps = ["StaticArraysCore"] -git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" -uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" -version = "1.1.0" - -[[deps.DiffRules]] -deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" -uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.15.1" - -[[deps.Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[deps.Distributions]] -deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] -git-tree-sha1 = "db40d3aff76ea6a3619fdd15a8c78299221a2394" -uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.97" - - [deps.Distributions.extensions] - DistributionsChainRulesCoreExt = "ChainRulesCore" - DistributionsDensityInterfaceExt = "DensityInterface" - - [deps.Distributions.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" - -[[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" - -[[deps.DualNumbers]] -deps = ["Calculus", "NaNMath", "SpecialFunctions"] -git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" -uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" -version = "0.6.8" - -[[deps.Enzyme]] -deps = ["CEnum", "EnzymeCore", "Enzyme_jll", "GPUCompiler", "LLVM", "Libdl", "LinearAlgebra", "ObjectFile", "Printf", "Random"] -git-tree-sha1 = "4ae2f569c382c5fd2374817820b6f914ca21106d" -uuid = "7da242da-08ed-463a-9acd-ee780be4f1d9" -version = "0.11.2" - -[[deps.EnzymeCore]] -deps = ["Adapt"] -git-tree-sha1 = "d0840cfff51e34729d20fd7d0a13938dc983878b" -uuid = "f151be2c-9106-41f4-ab19-57ee4f262869" -version = "0.3.0" - -[[deps.Enzyme_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "e155a67e28eec9884a95ba3fc920e4d23c20f95f" -uuid = "7cc45869-7501-5eee-bdea-0790c847d4ef" -version = "0.0.71+0" - -[[deps.ExceptionUnwrapping]] -deps = ["Test"] -git-tree-sha1 = "e90caa41f5a86296e014e148ee061bd6c3edec96" -uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" -version = "0.1.9" - -[[deps.Expat_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "4558ab818dcceaab612d1bb8c19cee87eda2b83c" -uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.5.0+0" - -[[deps.ExprTools]] -git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" -uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.9" - -[[deps.FFMPEG]] -deps = ["FFMPEG_jll"] -git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" -uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" -version = "0.4.1" - -[[deps.FFMPEG_jll]] -deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Pkg", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] -git-tree-sha1 = "74faea50c1d007c85837327f6775bea60b5492dd" -uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" -version = "4.4.2+2" - -[[deps.FLoops]] -deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] -git-tree-sha1 = "ffb97765602e3cbe59a0589d237bf07f245a8576" -uuid = "cc61a311-1640-44b5-9fba-1b764f453329" -version = "0.2.1" - -[[deps.FLoopsBase]] -deps = ["ContextVariablesX"] -git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" -uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" -version = "0.1.1" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" - -[[deps.FillArrays]] -deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "0b3b52afd0f87b0a3f5ada0466352d125c9db458" -uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.2.1" - -[[deps.FixedPointNumbers]] -deps = ["Statistics"] -git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" -uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -version = "0.8.4" - -[[deps.Flux]] -deps = ["Adapt", "CUDA", "ChainRulesCore", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "NNlibCUDA", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote", "cuDNN"] -git-tree-sha1 = "3e2c3704c2173ab4b1935362384ca878b53d4c34" -uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.13.17" - - [deps.Flux.extensions] - AMDGPUExt = "AMDGPU" - FluxMetalExt = "Metal" - - [deps.Flux.weakdeps] - AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" - Metal = "dde4c033-4e86-420c-a63e-0dd931031962" - -[[deps.Fontconfig_jll]] -deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03" -uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" -version = "2.13.93+0" - -[[deps.Formatting]] -deps = ["Printf"] -git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" -uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" -version = "0.4.2" - -[[deps.ForwardDiff]] -deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] -git-tree-sha1 = "00e252f4d706b3d55a8863432e742bf5717b498d" -uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.35" -weakdeps = ["StaticArrays"] - - [deps.ForwardDiff.extensions] - ForwardDiffStaticArraysExt = "StaticArrays" - -[[deps.FreeType2_jll]] -deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] -git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9" -uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" -version = "2.10.4+0" - -[[deps.FriBidi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91" -uuid = "559328eb-81f9-559d-9380-de523a88c83c" -version = "1.0.10+0" - -[[deps.FunctionChains]] -deps = ["Tricks"] -git-tree-sha1 = "fecc2a8863a4d63c26dadeffcdba7d1942334cee" -uuid = "8e6b2b91-af83-483e-ba35-d00930e4cf9b" -version = "0.1.4" - - [deps.FunctionChains.extensions] - FunctionChainsAdaptExt = "Adapt" - FunctionChainsChangesOfVariablesExt = "ChangesOfVariables" - FunctionChainsFlexiMapsExt = "FlexiMaps" - FunctionChainsFunctorsExt = "Functors" - FunctionChainsInverseFunctionsExt = "InverseFunctions" - - [deps.FunctionChains.weakdeps] - Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" - ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" - FlexiMaps = "6394faf6-06db-4fa8-b750-35ccc60383f7" - Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" - InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" - -[[deps.FunctionWrappers]] -git-tree-sha1 = "d62485945ce5ae9c0c48f124a84998d755bae00e" -uuid = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e" -version = "1.1.3" - -[[deps.Functors]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "478f8c3145bb91d82c2cf20433e8c1b30df454cc" -uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" -version = "0.4.4" - -[[deps.Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - -[[deps.GLFW_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"] -git-tree-sha1 = "d972031d28c8c8d9d7b41a536ad7bb0c2579caca" -uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89" -version = "3.3.8+0" - -[[deps.GPUArrays]] -deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "2e57b4a4f9cc15e85a24d603256fe08e527f48d1" -uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "8.8.1" - -[[deps.GPUArraysCore]] -deps = ["Adapt"] -git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" -uuid = "46192b85-c4d5-4398-a991-12ede77f4527" -version = "0.1.5" - -[[deps.GPUCompiler]] -deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "5737dc242dadd392d934ee330c69ceff47f0259c" -uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.19.4" - -[[deps.GR]] -deps = ["Artifacts", "Base64", "DelimitedFiles", "Downloads", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Preferences", "Printf", "Random", "Serialization", "Sockets", "TOML", "Tar", "Test", "UUIDs", "p7zip_jll"] -git-tree-sha1 = "8b8a2fd4536ece6e554168c21860b6820a8a83db" -uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" -version = "0.72.7" - -[[deps.GR_jll]] -deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Qt5Base_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "19fad9cd9ae44847fe842558a744748084a722d1" -uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" -version = "0.72.7+0" - -[[deps.Gettext_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] -git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" -uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" -version = "0.21.0+0" - -[[deps.Glib_jll]] -deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "d3b3624125c1474292d0d8ed0f65554ac37ddb23" -uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" -version = "2.74.0+2" - -[[deps.Graphite2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" -uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" -version = "1.3.14+0" - -[[deps.Grisu]] -git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" -uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" -version = "1.0.2" - -[[deps.HTTP]] -deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "2613d054b0e18a3dea99ca1594e9a3960e025da4" -uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.9.7" - -[[deps.HarfBuzz_jll]] -deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] -git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3" -uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" -version = "2.8.1+1" - -[[deps.HypergeometricFunctions]] -deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] -git-tree-sha1 = "0ec02c648befc2f94156eaef13b0f38106212f3f" -uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" -version = "0.3.17" - -[[deps.IRTools]] -deps = ["InteractiveUtils", "MacroTools", "Test"] -git-tree-sha1 = "eac00994ce3229a464c2847e956d77a2c64ad3a5" -uuid = "7869d1d1-7146-5819-86e3-90919afe41df" -version = "0.4.10" - -[[deps.InitialValues]] -git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" -uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" -version = "0.3.1" - -[[deps.InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[deps.InverseFunctions]] -deps = ["Test"] -git-tree-sha1 = "6667aadd1cdee2c6cd068128b3d226ebc4fb0c67" -uuid = "3587e190-3f89-42d0-90ee-14403ec27112" -version = "0.1.9" - -[[deps.IrrationalConstants]] -git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" -uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" -version = "0.2.2" - -[[deps.IteratorInterfaceExtensions]] -git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" -uuid = "82899510-4779-5014-852e-03e436cf321d" -version = "1.0.0" - -[[deps.JLFzf]] -deps = ["Pipe", "REPL", "Random", "fzf_jll"] -git-tree-sha1 = "f377670cda23b6b7c1c0b3893e37451c5c1a2185" -uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c" -version = "0.1.5" - -[[deps.JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.4.1" - -[[deps.JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.4" - -[[deps.JpegTurbo_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6f2675ef130a300a112286de91973805fcc5ffbc" -uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" -version = "2.1.91+0" - -[[deps.JuliaInterpreter]] -deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] -git-tree-sha1 = "6a125e6a4cb391e0b9adbd1afa9e771c2179f8ef" -uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" -version = "0.9.23" - -[[deps.JuliaVariables]] -deps = ["MLStyle", "NameResolution"] -git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" -uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" -version = "0.2.4" - -[[deps.KernelAbstractions]] -deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "b48617c5d764908b5fac493cd907cf33cc11eec1" -uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" -version = "0.9.6" - -[[deps.LAME_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" -uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" -version = "3.100.1+0" - -[[deps.LERC_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "bf36f528eec6634efc60d7ec062008f171071434" -uuid = "88015f11-f218-50d7-93a8-a6af411a945d" -version = "3.0.0+1" - -[[deps.LLVM]] -deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "5007c1421563108110bbd57f63d8ad4565808818" -uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "5.2.0" - -[[deps.LLVMExtra_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "1222116d7313cdefecf3d45a2bc1a89c4e7c9217" -uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -version = "0.0.22+0" - -[[deps.LLVMOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f689897ccbe049adb19a065c495e75f372ecd42b" -uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" -version = "15.0.4+0" - -[[deps.LZO_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" -uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" -version = "2.10.1+0" - -[[deps.LaTeXStrings]] -git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996" -uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" -version = "1.3.0" - -[[deps.Latexify]] -deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "OrderedCollections", "Printf", "Requires"] -git-tree-sha1 = "f428ae552340899a935973270b8d98e5a31c49fe" -uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" -version = "0.16.1" - - [deps.Latexify.extensions] - DataFramesExt = "DataFrames" - SymEngineExt = "SymEngine" - - [deps.Latexify.weakdeps] - DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" - SymEngine = "123dc426-2d89-5057-bbad-38513e3affd8" - -[[deps.LazyArtifacts]] -deps = ["Artifacts", "Pkg"] -uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.3" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "7.84.0+0" - -[[deps.LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.10.2+0" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[deps.Libffi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290" -uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" -version = "3.2.2+1" - -[[deps.Libgcrypt_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] -git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae" -uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" -version = "1.8.7+0" - -[[deps.Libglvnd_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"] -git-tree-sha1 = "6f73d1dd803986947b2c750138528a999a6c7733" -uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29" -version = "1.6.0+0" - -[[deps.Libgpg_error_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9" -uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" -version = "1.42.0+0" - -[[deps.Libiconv_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "c7cb1f5d892775ba13767a87c7ada0b980ea0a71" -uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.16.1+2" - -[[deps.Libmount_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" -uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" -version = "2.35.0+0" - -[[deps.Libtiff_jll]] -deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"] -git-tree-sha1 = "3eb79b0ca5764d4799c06699573fd8f533259713" -uuid = "89763e89-9b03-5906-acba-b20f662cd828" -version = "4.4.0+0" - -[[deps.Libuuid_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066" -uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" -version = "2.36.0+0" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[deps.LogExpFunctions]] -deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] -git-tree-sha1 = "c3ce8e7420b3a6e071e0fe4745f5d4300e37b13f" -uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" -version = "0.3.24" -weakdeps = ["ChainRulesCore", "ChangesOfVariables", "InverseFunctions"] - - [deps.LogExpFunctions.extensions] - LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" - LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" - LogExpFunctionsInverseFunctionsExt = "InverseFunctions" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[deps.LoggingExtras]] -deps = ["Dates", "Logging"] -git-tree-sha1 = "cedb76b37bc5a6c702ade66be44f831fa23c681e" -uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.0.0" - -[[deps.LoweredCodeUtils]] -deps = ["JuliaInterpreter"] -git-tree-sha1 = "60168780555f3e663c536500aa790b6368adc02a" -uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" -version = "2.3.0" - -[[deps.MLStyle]] -git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" -uuid = "d8e11817-5142-5d16-987a-aa16d5891078" -version = "0.4.17" - -[[deps.MLUtils]] -deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] -git-tree-sha1 = "3504cdb8c2bc05bde4d4b09a81b01df88fcbbba0" -uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" -version = "0.4.3" - -[[deps.MacroTools]] -deps = ["Markdown", "Random"] -git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.10" - -[[deps.MappedArrays]] -git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" -uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" -version = "0.4.2" - -[[deps.Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[deps.MbedTLS]] -deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "Random", "Sockets"] -git-tree-sha1 = "03a9b9718f5682ecb107ac9f7308991db4ce395b" -uuid = "739be429-bea8-5141-9913-cc70e7f3736d" -version = "1.1.7" - -[[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+0" - -[[deps.Measures]] -git-tree-sha1 = "c13304c81eec1ed3af7fc20e75fb6b26092a1102" -uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e" -version = "0.3.2" - -[[deps.MicroCollections]] -deps = ["BangBang", "InitialValues", "Setfield"] -git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" -uuid = "128add7d-3638-4c79-886c-908ea0c25c34" -version = "0.1.4" - -[[deps.Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.1.0" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.10.11" - -[[deps.NNlib]] -deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] -git-tree-sha1 = "72240e3f5ca031937bd536182cb2c031da5f46dd" -uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.8.21" - - [deps.NNlib.extensions] - NNlibAMDGPUExt = "AMDGPU" - - [deps.NNlib.weakdeps] - AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" - -[[deps.NNlibCUDA]] -deps = ["Adapt", "CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics", "cuDNN"] -git-tree-sha1 = "f94a9684394ff0d325cc12b06da7032d8be01aaf" -uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" -version = "0.2.7" - -[[deps.NaNMath]] -deps = ["OpenLibm_jll"] -git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" -uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "1.0.2" - -[[deps.NameResolution]] -deps = ["PrettyPrint"] -git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" -uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" -version = "0.1.5" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" - -[[deps.NormalizingFlows]] -deps = ["ADTypes", "Bijectors", "DiffResults", "Distributions", "DocStringExtensions", "Enzyme", "ForwardDiff", "LinearAlgebra", "Optimisers", "ProgressMeter", "Random", "ReverseDiff", "StatsBase", "Zygote"] -path = ".." -uuid = "50e4474d-9f12-44b7-af7a-91ab30ff6256" -version = "0.1.0" - -[[deps.ObjectFile]] -deps = ["Reexport", "StructIO"] -git-tree-sha1 = "55ce61d43409b1fb0279d1781bf3b0f22c83ab3b" -uuid = "d8793406-e978-5875-9003-1fc021f44a92" -version = "0.3.7" - -[[deps.Ogg_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f" -uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" -version = "1.3.5+1" - -[[deps.OneHotArrays]] -deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] -git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c" -uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" -version = "0.2.4" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.21+4" - -[[deps.OpenLibm_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+0" - -[[deps.OpenSSL]] -deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] -git-tree-sha1 = "51901a49222b09e3743c65b8847687ae5fc78eb2" -uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" -version = "1.4.1" - -[[deps.OpenSSL_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1aa4b74f80b01c6bc2b89992b861b5f210e665b5" -uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "1.1.21+0" - -[[deps.OpenSpecFun_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" -uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" -version = "0.5.5+0" - -[[deps.Optimisers]] -deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "6a01f65dd8583dee82eecc2a19b0ff21521aa749" -uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" -version = "0.2.18" - -[[deps.Opus_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720" -uuid = "91d4177d-7536-5919-b921-800302f37372" -version = "1.3.2+0" - -[[deps.OrderedCollections]] -git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.6.0" - -[[deps.PCRE2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.42.0+0" - -[[deps.PDMats]] -deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "67eae2738d63117a196f497d7db789821bce61d1" -uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.17" - -[[deps.Parsers]] -deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.7.1" - -[[deps.Pipe]] -git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d" -uuid = "b98c9c47-44ae-5843-9183-064241ee97a0" -version = "1.3.0" - -[[deps.Pixman_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] -git-tree-sha1 = "64779bc4c9784fee475689a1752ef4d5747c5e87" -uuid = "30392449-352a-5448-841d-b1acce4e97dc" -version = "0.42.2+0" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.9.0" - -[[deps.PlotThemes]] -deps = ["PlotUtils", "Statistics"] -git-tree-sha1 = "1f03a2d339f42dca4a4da149c7e15e9b896ad899" -uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a" -version = "3.1.0" - -[[deps.PlotUtils]] -deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "Statistics"] -git-tree-sha1 = "f92e1315dadf8c46561fb9396e525f7200cdc227" -uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043" -version = "1.3.5" - -[[deps.Plots]] -deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Preferences", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun", "UnitfulLatexify", "Unzip"] -git-tree-sha1 = "75ca67b2c6512ad2d0c767a7cfc55e75075f8bbc" -uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.38.16" - - [deps.Plots.extensions] - FileIOExt = "FileIO" - GeometryBasicsExt = "GeometryBasics" - IJuliaExt = "IJulia" - ImageInTerminalExt = "ImageInTerminal" - UnitfulExt = "Unitful" - - [deps.Plots.weakdeps] - FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" - GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326" - IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" - ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" - Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" - -[[deps.PrecompileTools]] -deps = ["Preferences"] -git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" -uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.1.2" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.0" - -[[deps.PrettyPrint]] -git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" -uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" -version = "0.2.0" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[deps.Profile]] -deps = ["Printf"] -uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" - -[[deps.ProgressLogging]] -deps = ["Logging", "SHA", "UUIDs"] -git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" -uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" -version = "0.1.4" - -[[deps.ProgressMeter]] -deps = ["Distributed", "Printf"] -git-tree-sha1 = "d7a7aef8f8f2d537104f170139553b14dfe39fe9" -uuid = "92933f4c-e287-5a05-a399-4b506db050ca" -version = "1.7.2" - -[[deps.Qt5Base_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"] -git-tree-sha1 = "0c03844e2231e12fda4d0086fd7cbe4098ee8dc5" -uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" -version = "5.15.3+2" - -[[deps.QuadGK]] -deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" -uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.8.2" - -[[deps.REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[deps.Random]] -deps = ["SHA", "Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[deps.Random123]] -deps = ["Random", "RandomNumbers"] -git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" -uuid = "74087812-796a-5b5d-8853-05524746bad3" -version = "1.6.1" - -[[deps.RandomNumbers]] -deps = ["Random", "Requires"] -git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" -uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" -version = "1.5.3" - -[[deps.RealDot]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" -uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" -version = "0.1.0" - -[[deps.RecipesBase]] -deps = ["PrecompileTools"] -git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" -uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -version = "1.3.4" - -[[deps.RecipesPipeline]] -deps = ["Dates", "NaNMath", "PlotUtils", "PrecompileTools", "RecipesBase"] -git-tree-sha1 = "45cf9fd0ca5839d06ef333c8201714e888486342" -uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" -version = "0.6.12" - -[[deps.Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[deps.RelocatableFolders]] -deps = ["SHA", "Scratch"] -git-tree-sha1 = "90bc7a7c96410424509e4263e277e43250c05691" -uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" -version = "1.0.0" - -[[deps.Requires]] -deps = ["UUIDs"] -git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.0" - -[[deps.ReverseDiff]] -deps = ["ChainRulesCore", "DiffResults", "DiffRules", "ForwardDiff", "FunctionWrappers", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "Random", "SpecialFunctions", "StaticArrays", "Statistics"] -git-tree-sha1 = "5fb7c1053046a80b53cf9c0ef8526ea656de7b8c" -uuid = "37e2e3b7-166d-5795-8a7a-e32c996b4267" -version = "1.14.6" - -[[deps.Revise]] -deps = ["CodeTracking", "Distributed", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Pkg", "REPL", "Requires", "UUIDs", "Unicode"] -git-tree-sha1 = "1e597b93700fa4045d7189afa7c004e0584ea548" -uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" -version = "3.5.3" - -[[deps.Rmath]] -deps = ["Random", "Rmath_jll"] -git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" -uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" -version = "0.7.1" - -[[deps.Rmath_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" -uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" -version = "0.4.0+0" - -[[deps.Roots]] -deps = ["ChainRulesCore", "CommonSolve", "Printf", "Setfield"] -git-tree-sha1 = "de432823e8aab4dd1a985be4be768f95acf152d4" -uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" -version = "2.0.17" - - [deps.Roots.extensions] - RootsForwardDiffExt = "ForwardDiff" - RootsIntervalRootFindingExt = "IntervalRootFinding" - RootsSymPyExt = "SymPy" - - [deps.Roots.weakdeps] - ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" - IntervalRootFinding = "d2bf35a9-74e0-55ec-b149-d360ff49b807" - SymPy = "24249f21-da20-56a4-8eb1-6a02cf4ae2e6" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Scratch]] -deps = ["Dates"] -git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" -uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.0" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[deps.Setfield]] -deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] -git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" -uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" -version = "1.1.1" - -[[deps.ShowCases]] -git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" -uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" -version = "0.1.0" - -[[deps.Showoff]] -deps = ["Dates", "Grisu"] -git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" -uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" -version = "1.0.3" - -[[deps.SimpleBufferStream]] -git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" -uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" -version = "1.1.0" - -[[deps.SimpleTraits]] -deps = ["InteractiveUtils", "MacroTools"] -git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" -uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" -version = "0.9.4" - -[[deps.Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[deps.SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "c60ec5c62180f27efea3ba2908480f8055e17cee" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.1.1" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[deps.SpecialFunctions]] -deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "7beb031cf8145577fbccacd94b8a8f4ce78428d3" -uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.3.0" -weakdeps = ["ChainRulesCore"] - - [deps.SpecialFunctions.extensions] - SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" - -[[deps.SplittablesBase]] -deps = ["Setfield", "Test"] -git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" -uuid = "171d559e-b47b-412a-8079-5efa626c420e" -version = "0.1.15" - -[[deps.StaticArrays]] -deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] -git-tree-sha1 = "832afbae2a45b4ae7e831f86965469a24d1d8a83" -uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.5.26" - -[[deps.StaticArraysCore]] -git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" -uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" -version = "1.4.0" - -[[deps.Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.9.0" - -[[deps.StatsAPI]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" -uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.6.0" - -[[deps.StatsBase]] -deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" -uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.34.0" - -[[deps.StatsFuns]] -deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] -git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" -uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "1.3.0" -weakdeps = ["ChainRulesCore", "InverseFunctions"] - - [deps.StatsFuns.extensions] - StatsFunsChainRulesCoreExt = "ChainRulesCore" - StatsFunsInverseFunctionsExt = "InverseFunctions" - -[[deps.StructArrays]] -deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] -git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" -uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.6.15" - -[[deps.StructIO]] -deps = ["Test"] -git-tree-sha1 = "010dc73c7146869c042b49adcdb6bf528c12e859" -uuid = "53d494c1-5632-5724-8f4c-31dff12d585f" -version = "0.3.0" - -[[deps.SuiteSparse]] -deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] -uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "5.10.1+6" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.TableTraits]] -deps = ["IteratorInterfaceExtensions"] -git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" -uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" -version = "1.0.1" - -[[deps.Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] -git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" -uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.10.1" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.TensorCore]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" -uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" -version = "0.1.1" - -[[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.TimerOutputs]] -deps = ["ExprTools", "Printf"] -git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" -uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.23" - -[[deps.TranscodingStreams]] -deps = ["Random", "Test"] -git-tree-sha1 = "9a6ae7ed916312b41236fcef7e0af564ef934769" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.13" - -[[deps.Transducers]] -deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] -git-tree-sha1 = "a66fb81baec325cf6ccafa243af573b031e87b00" -uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" -version = "0.4.77" - - [deps.Transducers.extensions] - TransducersBlockArraysExt = "BlockArrays" - TransducersDataFramesExt = "DataFrames" - TransducersLazyArraysExt = "LazyArrays" - TransducersOnlineStatsBaseExt = "OnlineStatsBase" - TransducersReferenceablesExt = "Referenceables" - - [deps.Transducers.weakdeps] - BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" - DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" - LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" - OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" - Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" - -[[deps.Tricks]] -git-tree-sha1 = "aadb748be58b492045b4f56166b5188aa63ce549" -uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775" -version = "0.1.7" - -[[deps.URIs]] -git-tree-sha1 = "074f993b0ca030848b897beff716d93aca60f06a" -uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.4.2" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[deps.UnicodeFun]] -deps = ["REPL"] -git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" -uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" -version = "0.4.1" - -[[deps.Unitful]] -deps = ["ConstructionBase", "Dates", "LinearAlgebra", "Random"] -git-tree-sha1 = "ba4aa36b2d5c98d6ed1f149da916b3ba46527b2b" -uuid = "1986cc42-f94f-5a68-af5c-568840ba703d" -version = "1.14.0" -weakdeps = ["InverseFunctions"] - - [deps.Unitful.extensions] - InverseFunctionsUnitfulExt = "InverseFunctions" - -[[deps.UnitfulLatexify]] -deps = ["LaTeXStrings", "Latexify", "Unitful"] -git-tree-sha1 = "e2d817cc500e960fdbafcf988ac8436ba3208bfd" -uuid = "45397f5d-5981-4c77-b2b3-fc36d6e9b728" -version = "1.6.3" - -[[deps.UnsafeAtomics]] -git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" -uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" -version = "0.2.1" - -[[deps.UnsafeAtomicsLLVM]] -deps = ["LLVM", "UnsafeAtomics"] -git-tree-sha1 = "ea37e6066bf194ab78f4e747f5245261f17a7175" -uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" -version = "0.1.2" - -[[deps.Unzip]] -git-tree-sha1 = "ca0969166a028236229f63514992fc073799bb78" -uuid = "41fe7b60-77ed-43a1-b4f0-825fd5a5650d" -version = "0.2.0" - -[[deps.Wayland_jll]] -deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] -git-tree-sha1 = "ed8d92d9774b077c53e1da50fd81a36af3744c1c" -uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" -version = "1.21.0+0" - -[[deps.Wayland_protocols_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4528479aa01ee1b3b4cd0e6faef0e04cf16466da" -uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91" -version = "1.25.0+0" - -[[deps.XML2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "93c41695bc1c08c46c5899f4fe06d6ead504bb73" -uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.10.3+0" - -[[deps.XSLT_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] -git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" -uuid = "aed1982a-8fda-507f-9586-7b0439959a61" -version = "1.1.34+0" - -[[deps.Xorg_libX11_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] -git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527" -uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" -version = "1.6.9+4" - -[[deps.Xorg_libXau_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e" -uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" -version = "1.0.9+4" - -[[deps.Xorg_libXcursor_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"] -git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd" -uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724" -version = "1.2.0+4" - -[[deps.Xorg_libXdmcp_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4" -uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" -version = "1.1.3+4" - -[[deps.Xorg_libXext_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" -uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" -version = "1.3.4+4" - -[[deps.Xorg_libXfixes_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4" -uuid = "d091e8ba-531a-589c-9de9-94069b037ed8" -version = "5.0.3+4" - -[[deps.Xorg_libXi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"] -git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246" -uuid = "a51aa0fd-4e3c-5386-b890-e753decda492" -version = "1.7.10+4" - -[[deps.Xorg_libXinerama_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"] -git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123" -uuid = "d1454406-59df-5ea1-beac-c340f2130bc3" -version = "1.1.4+4" - -[[deps.Xorg_libXrandr_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"] -git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631" -uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484" -version = "1.5.2+4" - -[[deps.Xorg_libXrender_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" -uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" -version = "0.9.10+4" - -[[deps.Xorg_libpthread_stubs_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb" -uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" -version = "0.1.0+3" - -[[deps.Xorg_libxcb_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] -git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6" -uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" -version = "1.13.0+3" - -[[deps.Xorg_libxkbfile_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2" -uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a" -version = "1.1.0+4" - -[[deps.Xorg_xcb_util_image_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97" -uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b" -version = "0.4.0+1" - -[[deps.Xorg_xcb_util_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"] -git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1" -uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5" -version = "0.4.0+1" - -[[deps.Xorg_xcb_util_keysyms_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00" -uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7" -version = "0.4.0+1" - -[[deps.Xorg_xcb_util_renderutil_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e" -uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e" -version = "0.3.9+1" - -[[deps.Xorg_xcb_util_wm_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67" -uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361" -version = "0.4.1+1" - -[[deps.Xorg_xkbcomp_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"] -git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b" -uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4" -version = "1.4.2+4" - -[[deps.Xorg_xkeyboard_config_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"] -git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d" -uuid = "33bec58e-1273-512f-9401-5d533626f822" -version = "2.27.0+4" - -[[deps.Xorg_xtrans_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845" -uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" -version = "1.4.0+3" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+0" - -[[deps.Zstd_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "49ce682769cd5de6c72dcf1b94ed7790cd08974c" -uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" -version = "1.5.5+0" - -[[deps.Zygote]] -deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "5be3ddb88fc992a7d8ea96c3f10a49a7e98ebc7b" -uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.62" - - [deps.Zygote.extensions] - ZygoteColorsExt = "Colors" - ZygoteDistancesExt = "Distances" - ZygoteTrackerExt = "Tracker" - - [deps.Zygote.weakdeps] - Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" - Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" - Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" - -[[deps.ZygoteRules]] -deps = ["ChainRulesCore", "MacroTools"] -git-tree-sha1 = "977aed5d006b840e2e40c0b48984f7463109046d" -uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.2.3" - -[[deps.cuDNN]] -deps = ["CEnum", "CUDA", "CUDNN_jll"] -git-tree-sha1 = "ec954b59f6b0324543f2e3ed8118309ac60cb75b" -uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" -version = "1.0.3" - -[[deps.fzf_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "868e669ccb12ba16eaf50cb2957ee2ff61261c56" -uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09" -version = "0.29.0+0" - -[[deps.libaom_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "3a2ea60308f0996d26f1e5354e10c24e9ef905d4" -uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" -version = "3.4.0+0" - -[[deps.libass_jll]] -deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] -git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47" -uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" -version = "0.15.1+0" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.8.0+0" - -[[deps.libfdk_aac_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55" -uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" -version = "2.0.2+0" - -[[deps.libpng_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] -git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c" -uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" -version = "1.6.38+0" - -[[deps.libvorbis_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] -git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c" -uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" -version = "1.3.7+1" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.48.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+0" - -[[deps.x264_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" -uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" -version = "2021.5.5+0" - -[[deps.x265_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" -uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" -version = "3.5.0+0" - -[[deps.xkbcommon_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] -git-tree-sha1 = "9ebfc140cc56e8c2156a15ceac2f0302e327ac0a" -uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" -version = "1.4.1+0" From e4f5efac40999e524a3666d2d5592fdafa30b83d Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 10:58:55 -0700 Subject: [PATCH 04/40] minor bug fix for trainig loop --- src/train.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/train.jl b/src/train.jl index a8e07a45..d76e6f6e 100644 --- a/src/train.jl +++ b/src/train.jl @@ -178,7 +178,7 @@ function optimize( # initialise optimiser state st = Optimisers.setup(optimiser, θ) - # TODO: Add support for general `hasconverged(...)` approach to allow early termination. + # general `hasconverged(...)` approach to allow early termination. converged = false i = 1 time_elapsed = @elapsed while (i ≤ max_iters) && !converged @@ -201,11 +201,11 @@ function optimize( st, θ = Optimisers.update!(st, θ, DiffResults.gradient(diff_result)) # check convergence - converged = hasconverged(i, opt_stats, re, θ, st) i += 1 + converged = hasconverged(i, stat, re, θ, st) pm_next!(prog, stat) end # return status of the optimiser for potential continuation of training - return θ, map(identity, opt_stats), st + return θ, map(identity, opt_stats[1:(i - 1)]), st end From b345e7801c41256bbfc1dbe48cd0acc7dd700adf Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 10:59:16 -0700 Subject: [PATCH 05/40] test new stopping criterion --- test/interface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/interface.jl b/test/interface.jl index 14a47bc7..ef47690c 100644 --- a/test/interface.jl +++ b/test/interface.jl @@ -20,7 +20,7 @@ sample_per_iter = 10 cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) - checkconv(iter, opt_stats, re, θ, st) = iter > 4999 + checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 flow_trained, stats, _ = train_flow( elbo, flow, From f10512fb108c526af395253127c9bdb73dbd1e9f Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 11:00:19 -0700 Subject: [PATCH 06/40] test convergent condition/ rm unready examples --- example/neural_spline_flow/example.jl | 22 ---------------------- example/planar_radial_flow/planar_flow.jl | 2 ++ 2 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 example/neural_spline_flow/example.jl diff --git a/example/neural_spline_flow/example.jl b/example/neural_spline_flow/example.jl deleted file mode 100644 index d54fce71..00000000 --- a/example/neural_spline_flow/example.jl +++ /dev/null @@ -1,22 +0,0 @@ -using Random, Distributions, LinearAlgebra, Bijectors -using ADTypes -using Optimisers -using FunctionChains -using NormalizingFlows -using Flux: f32 -using Plots -include("../common.jl") - -Random.seed!(123) -rng = Random.default_rng() - -###################################### -# 2d Banana as the target distribution -###################################### -include("../targets/banana.jl") - -# create target p -p = Banana(2, 1.0f-1, 100.0f0) -logp = Base.Fix1(logpdf, p) - -# work in progress, not for review \ No newline at end of file diff --git a/example/planar_radial_flow/planar_flow.jl b/example/planar_radial_flow/planar_flow.jl index 0ef69984..2e26ffe0 100644 --- a/example/planar_radial_flow/planar_flow.jl +++ b/example/planar_radial_flow/planar_flow.jl @@ -36,6 +36,7 @@ flow_untrained = deepcopy(flow) # train the flow sample_per_iter = 10 cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 flow_trained, stats, _ = train_flow( elbo, flow, @@ -44,6 +45,7 @@ flow_trained, stats, _ = train_flow( max_iters=200_00, optimiser=Optimisers.ADAM(), callback=cb, + hasconverged=checkconv, ) losses = map(x -> x.loss, stats) From 1c1c88a89347ff962465122ac616c59cbf2d70f6 Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 11:24:05 -0700 Subject: [PATCH 07/40] rm julia test from CI --- .github/workflows/CI.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index cde6178f..76013706 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,7 +18,6 @@ jobs: fail-fast: false matrix: version: - - '1' - '1.6' os: - ubuntu-latest From 5d2844fccdc1a3183ed4b93d0b6b80d5764a15ef Mon Sep 17 00:00:00 2001 From: zuheng Date: Tue, 11 Jul 2023 12:12:36 -0700 Subject: [PATCH 08/40] Revert "rm julia test from CI" This reverts commit 1c1c88a89347ff962465122ac616c59cbf2d70f6. --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 76013706..cde6178f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,6 +18,7 @@ jobs: fail-fast: false matrix: version: + - '1' - '1.6' os: - ubuntu-latest From 3fdcb0e73eb8f0c25ee496ad002d216556b5b276 Mon Sep 17 00:00:00 2001 From: zuheng Date: Wed, 12 Jul 2023 14:28:25 -0700 Subject: [PATCH 09/40] make autodiff pkgs as extension + require for bwd compat --- Project.toml | 10 +++--- ext/NormalizingFlowsEnzymeExt.jl | 27 +++++++++++++++ ext/NormalizingFlowsForwardDiffExt.jl | 31 +++++++++++++++++ ext/NormalizingFlowsReverseDiffExt.jl | 23 +++++++++++++ ext/NormalizingFlowsZygoteExt.jl | 26 +++++++++++++++ src/NormalizingFlows.jl | 24 +++++++++++++- src/train.jl | 48 --------------------------- test/Project.toml | 4 +++ test/runtests.jl | 1 + 9 files changed, 141 insertions(+), 53 deletions(-) create mode 100644 ext/NormalizingFlowsEnzymeExt.jl create mode 100644 ext/NormalizingFlowsForwardDiffExt.jl create mode 100644 ext/NormalizingFlowsReverseDiffExt.jl create mode 100644 ext/NormalizingFlowsZygoteExt.jl diff --git a/Project.toml b/Project.toml index af9f74bc..874fecb0 100644 --- a/Project.toml +++ b/Project.toml @@ -8,14 +8,17 @@ Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" DiffResults = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" + +[weakdeps] +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] @@ -30,4 +33,3 @@ ReverseDiff = "1.14" StatsBase = "0.33, 0.34" Zygote = "0.6" julia = "1.6" - diff --git a/ext/NormalizingFlowsEnzymeExt.jl b/ext/NormalizingFlowsEnzymeExt.jl new file mode 100644 index 00000000..51b7fe34 --- /dev/null +++ b/ext/NormalizingFlowsEnzymeExt.jl @@ -0,0 +1,27 @@ +module NormalizingFlowsEnzymeExt + +if isdefined(Base, :get_extension) + using Enzyme + using NormalizingFlows: value_and_gradient! + using ADTypes + using DiffResults +else + using ..Enzyme + using ..NormalizingFlows: value_and_gradient! + using ..ADTypes + using ..DiffResults +end + +# Enzyme doesn't support f::Bijectors (see https://github.com/EnzymeAD/Enzyme.jl/issues/916) +function value_and_gradient!( + ad::ADTypes.AutoEnzyme, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult +) where {T<:Real} + y = f(θ) + DiffResults.value!(out, y) + ∇θ = DiffResults.gradient(out) + fill!(∇θ, zero(T)) + Enzyme.autodiff(Enzyme.ReverseWithPrimal, f, Enzyme.Active, Enzyme.Duplicated(θ, ∇θ)) + return out +end + +end \ No newline at end of file diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl new file mode 100644 index 00000000..fe42b1d3 --- /dev/null +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -0,0 +1,31 @@ +module NormalizingFlowsForwardDiffExt + +if isdefined(Base, :get_extension) + using ForwardDiff + using NormalizingFlows: value_and_gradient! + using ADTypes + using DiffResults +else + using ..ForwardDiff + using ..NormalizingFlows: value_and_gradient! + using ..ADTypes + using ..DiffResults +end + +# ForwardDiff +# extract chunk size from AutoForwardDiff +getchunksize(::ADTypes.AutoForwardDiff{chunksize}) where {chunksize} = chunksize +function value_and_gradient!( + ad::ADTypes.AutoForwardDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult +) where {T<:Real} + chunk_size = getchunksize(ad) + config = if isnothing(chunk_size) + ForwardDiff.GradientConfig(f, θ) + else + ForwardDiff.GradientConfig(f, θ, ForwardDiff.Chunk(length(θ), chunk_size)) + end + ForwardDiff.gradient!(out, f, θ, config) + return out +end + +end \ No newline at end of file diff --git a/ext/NormalizingFlowsReverseDiffExt.jl b/ext/NormalizingFlowsReverseDiffExt.jl new file mode 100644 index 00000000..0837068b --- /dev/null +++ b/ext/NormalizingFlowsReverseDiffExt.jl @@ -0,0 +1,23 @@ +module NormalizingFlowsReverseDiffExt + +if isdefined(Base, :get_extension) + using ReverseDiff + using NormalizingFlows: value_and_gradient! + using ADTypes + using DiffResults +else + using ..ReverseDiff + using ..NormalizingFlows: value_and_gradient! + using ..ADTypes + using ..DiffResults +end + +# ReverseDiff without compiled tape +function value_and_gradient!( + ad::ADTypes.AutoReverseDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult +) where {T<:Real} + tp = ReverseDiff.GradientTape(f, θ) + ReverseDiff.gradient!(out, tp, θ) + return out +end +end \ No newline at end of file diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl new file mode 100644 index 00000000..c5712553 --- /dev/null +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -0,0 +1,26 @@ +module NormalizingFlowsZygoteExt + +if isdefined(Base, :get_extension) + using Zygote + using NormalizingFlows: value_and_gradient! + using ADTypes + using DiffResults +else + using ..Zygote + using ..NormalizingFlows: value_and_gradient! + using ..ADTypes + using ..DiffResults +end + +# zygote +function value_and_gradient!( + ad::ADTypes.AutoZygote, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult +) where {T<:Real} + y, back = Zygote.pullback(f, θ) + ∇θ = back(one(T)) + DiffResults.value!(out, y) + DiffResults.gradient!(out, first(∇θ)) + return out +end + +end \ No newline at end of file diff --git a/src/NormalizingFlows.jl b/src/NormalizingFlows.jl index 5b289aa1..403f3207 100644 --- a/src/NormalizingFlows.jl +++ b/src/NormalizingFlows.jl @@ -5,7 +5,6 @@ using Optimisers using LinearAlgebra, Random, Distributions, StatsBase using ProgressMeter using ADTypes, DiffResults -using Zygote, ForwardDiff, ReverseDiff, Enzyme using DocStringExtensions @@ -72,4 +71,27 @@ end include("train.jl") include("objectives.jl") +# optional dependencies +if !isdefined(Base, :get_extension) # check whether :get_extension is defined in Base + using Requires +end + +# Question: should Exts be loaded here or in train.jl? +function __init__() + @static if !isdefined(Base, :get_extension) + @require ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" include( + "../ext/NormalizingFlowsForwardDiffExt.jl" + ) + @require ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" include( + "../ext/NormalizingFlowsReverseDiffExt.jl" + ) + @require Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" include( + "../ext/NormalizingFlowsEnzymeExt.jl" + ) + @require Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" include( + "../ext/NormalizingFlowsZygoteExt.jl" + ) + end +end + end diff --git a/src/train.jl b/src/train.jl index d76e6f6e..2ca82421 100644 --- a/src/train.jl +++ b/src/train.jl @@ -12,54 +12,6 @@ The function `f` must return a scalar value. The gradient is stored in `out` as vector of the same length as `θ`. """ function value_and_gradient! end -# TODO: Make these definitions extensions to avoid loading unneecssary packages. -# zygote -function value_and_gradient!( - ad::ADTypes.AutoZygote, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult -) where {T<:Real} - y, back = Zygote.pullback(f, θ) - ∇θ = back(one(T)) - DiffResults.value!(out, y) - DiffResults.gradient!(out, first(∇θ)) - return out -end - -# ForwardDiff -# extract chunk size from AutoForwardDiff -getchunksize(::ADTypes.AutoForwardDiff{chunksize}) where {chunksize} = chunksize -function value_and_gradient!( - ad::ADTypes.AutoForwardDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult -) where {T<:Real} - chunk_size = getchunksize(ad) - config = if isnothing(chunk_size) - ForwardDiff.GradientConfig(f, θ) - else - ForwardDiff.GradientConfig(f, θ, ForwardDiff.Chunk(length(θ), chunk_size)) - end - ForwardDiff.gradient!(out, f, θ, config) - return out -end - -# ReverseDiff without compiled tape -function value_and_gradient!( - ad::ADTypes.AutoReverseDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult -) where {T<:Real} - tp = ReverseDiff.GradientTape(f, θ) - ReverseDiff.gradient!(out, tp, θ) - return out -end - -# Enzyme -function value_and_gradient!( - ad::ADTypes.AutoEnzyme, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult -) where {T<:Real} - y = f(θ) - DiffResults.value!(out, y) - ∇θ = DiffResults.gradient(out) - fill!(∇θ, zero(T)) - Enzyme.autodiff(Enzyme.ReverseWithPrimal, f, Enzyme.Active, Enzyme.Duplicated(θ, ∇θ)) - return out -end """ grad!( diff --git a/test/Project.toml b/test/Project.toml index 0bed305d..c474adda 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -3,7 +3,11 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" DiffResults = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/test/runtests.jl b/test/runtests.jl index 385daeeb..e050a645 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using Bijectors, Optimisers using LinearAlgebra using Random using ADTypes, DiffResults +using ForwardDiff, Zygote, Enzyme, ReverseDiff using Test include("ad.jl") From ddad59e9ac3eba85d4d7725b45173cc45fd908ac Mon Sep 17 00:00:00 2001 From: zuheng Date: Wed, 12 Jul 2023 16:15:32 -0700 Subject: [PATCH 10/40] debugging Ext --- example/Project.toml | 1 + example/planar_radial_flow/planar_flow.jl | 4 ++-- example/planar_radial_flow/radial_flow.jl | 1 + src/NormalizingFlows.jl | 5 ++--- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/example/Project.toml b/example/Project.toml index cd7b7958..d462c5ee 100644 --- a/example/Project.toml +++ b/example/Project.toml @@ -14,3 +14,4 @@ Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/example/planar_radial_flow/planar_flow.jl b/example/planar_radial_flow/planar_flow.jl index 2e26ffe0..d1ccff81 100644 --- a/example/planar_radial_flow/planar_flow.jl +++ b/example/planar_radial_flow/planar_flow.jl @@ -3,6 +3,7 @@ using ADTypes using Optimisers using FunctionChains using NormalizingFlows +using Zygote using Flux: f32 using Plots include("../common.jl") @@ -36,7 +37,6 @@ flow_untrained = deepcopy(flow) # train the flow sample_per_iter = 10 cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) -checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 flow_trained, stats, _ = train_flow( elbo, flow, @@ -45,7 +45,7 @@ flow_trained, stats, _ = train_flow( max_iters=200_00, optimiser=Optimisers.ADAM(), callback=cb, - hasconverged=checkconv, + ADbackend=AutoZygote(), ) losses = map(x -> x.loss, stats) diff --git a/example/planar_radial_flow/radial_flow.jl b/example/planar_radial_flow/radial_flow.jl index 54f289f9..4d51cd24 100644 --- a/example/planar_radial_flow/radial_flow.jl +++ b/example/planar_radial_flow/radial_flow.jl @@ -3,6 +3,7 @@ using ADTypes using Optimisers using FunctionChains using NormalizingFlows +using Zygote using Flux: f32 using Plots include("../common.jl") diff --git a/src/NormalizingFlows.jl b/src/NormalizingFlows.jl index 403f3207..91457b8e 100644 --- a/src/NormalizingFlows.jl +++ b/src/NormalizingFlows.jl @@ -8,7 +8,7 @@ using ADTypes, DiffResults using DocStringExtensions -export train_flow, elbo, loglikelihood +export train_flow, elbo, loglikelihood, value_and_gradient! """ train_flow([rng::AbstractRNG, ]vo, flow, args...; kwargs...) @@ -43,7 +43,7 @@ function train_flow( args...; max_iters::Int=1000, optimiser::Optimisers.AbstractRule=Optimisers.ADAM(), - ADbackend::ADTypes.AbstractADType=ADTypes.AutoZygote(), + ADbackend::ADTypes.AbstractADType, kwargs..., ) # destruct flow for explicit access to the parameters @@ -93,5 +93,4 @@ function __init__() ) end end - end From ef60ee1f9316d7578fefa6c568f825e8b3e2f65e Mon Sep 17 00:00:00 2001 From: zuheng Date: Wed, 12 Jul 2023 21:39:32 -0700 Subject: [PATCH 11/40] keep debugging --- Project.toml | 14 ++++++++++++++ ext/NormalizingFlowsForwardDiffExt.jl | 1 - ext/NormalizingFlowsReverseDiffExt.jl | 1 + ext/NormalizingFlowsZygoteExt.jl | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 874fecb0..ace3e688 100644 --- a/Project.toml +++ b/Project.toml @@ -21,6 +21,20 @@ ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[extensions] +NormalizingFlowsEnzymeExt = ["Enzyme", "DiffResults", "ADTypes"] +NormalizingFlowsForwardDiffExt = ["ForwardDiff", "DiffResults", "ADTypes"] +NormalizingFlowsReverseDiffExt = ["ReverseDiff", "DiffResults", "ADTypes"] +NormalizingFlowsZygoteExt = ["Zygote", "DiffResults", "ADTypes"] + +[extras] +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + + [compat] ADTypes = "0.1" Bijectors = "0.12.6, 0.13" diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl index fe42b1d3..c873c5bd 100644 --- a/ext/NormalizingFlowsForwardDiffExt.jl +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -12,7 +12,6 @@ else using ..DiffResults end -# ForwardDiff # extract chunk size from AutoForwardDiff getchunksize(::ADTypes.AutoForwardDiff{chunksize}) where {chunksize} = chunksize function value_and_gradient!( diff --git a/ext/NormalizingFlowsReverseDiffExt.jl b/ext/NormalizingFlowsReverseDiffExt.jl index 0837068b..4bd444d1 100644 --- a/ext/NormalizingFlowsReverseDiffExt.jl +++ b/ext/NormalizingFlowsReverseDiffExt.jl @@ -20,4 +20,5 @@ function value_and_gradient!( ReverseDiff.gradient!(out, tp, θ) return out end + end \ No newline at end of file diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl index c5712553..ab87be68 100644 --- a/ext/NormalizingFlowsZygoteExt.jl +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -12,7 +12,7 @@ else using ..DiffResults end -# zygote +@info "loading zygote ext" function value_and_gradient!( ad::ADTypes.AutoZygote, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} From 5a5deb0789645089e52dee6ed22778cab26315b9 Mon Sep 17 00:00:00 2001 From: Xianda Sun Date: Thu, 13 Jul 2023 09:20:15 +0100 Subject: [PATCH 12/40] Fix AD package extension loading issues --- Project.toml | 10 ++++------ ext/NormalizingFlowsEnzymeExt.jl | 10 +++++----- ext/NormalizingFlowsForwardDiffExt.jl | 10 +++++----- ext/NormalizingFlowsReverseDiffExt.jl | 10 +++++----- ext/NormalizingFlowsZygoteExt.jl | 10 +++++----- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index ace3e688..cf824957 100644 --- a/Project.toml +++ b/Project.toml @@ -21,12 +21,11 @@ ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" - [extensions] -NormalizingFlowsEnzymeExt = ["Enzyme", "DiffResults", "ADTypes"] -NormalizingFlowsForwardDiffExt = ["ForwardDiff", "DiffResults", "ADTypes"] -NormalizingFlowsReverseDiffExt = ["ReverseDiff", "DiffResults", "ADTypes"] -NormalizingFlowsZygoteExt = ["Zygote", "DiffResults", "ADTypes"] +NormalizingFlowsEnzymeExt = "Enzyme" +NormalizingFlowsForwardDiffExt = "ForwardDiff" +NormalizingFlowsReverseDiffExt = "ReverseDiff" +NormalizingFlowsZygoteExt = "Zygote" [extras] Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" @@ -34,7 +33,6 @@ ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" - [compat] ADTypes = "0.1" Bijectors = "0.12.6, 0.13" diff --git a/ext/NormalizingFlowsEnzymeExt.jl b/ext/NormalizingFlowsEnzymeExt.jl index 51b7fe34..0542c669 100644 --- a/ext/NormalizingFlowsEnzymeExt.jl +++ b/ext/NormalizingFlowsEnzymeExt.jl @@ -1,19 +1,19 @@ module NormalizingFlowsEnzymeExt if isdefined(Base, :get_extension) - using Enzyme - using NormalizingFlows: value_and_gradient! using ADTypes using DiffResults + using Enzyme + using NormalizingFlows else - using ..Enzyme - using ..NormalizingFlows: value_and_gradient! using ..ADTypes using ..DiffResults + using ..Enzyme + using ..NormalizingFlows end # Enzyme doesn't support f::Bijectors (see https://github.com/EnzymeAD/Enzyme.jl/issues/916) -function value_and_gradient!( +function NormalizingFlows.value_and_gradient!( ad::ADTypes.AutoEnzyme, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} y = f(θ) diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl index c873c5bd..c4f70200 100644 --- a/ext/NormalizingFlowsForwardDiffExt.jl +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -1,20 +1,20 @@ module NormalizingFlowsForwardDiffExt if isdefined(Base, :get_extension) - using ForwardDiff - using NormalizingFlows: value_and_gradient! using ADTypes using DiffResults + using ForwardDiff + using NormalizingFlows else - using ..ForwardDiff - using ..NormalizingFlows: value_and_gradient! using ..ADTypes using ..DiffResults + using ..ForwardDiff + using ..NormalizingFlows end # extract chunk size from AutoForwardDiff getchunksize(::ADTypes.AutoForwardDiff{chunksize}) where {chunksize} = chunksize -function value_and_gradient!( +function NormalizingFlows.value_and_gradient!( ad::ADTypes.AutoForwardDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} chunk_size = getchunksize(ad) diff --git a/ext/NormalizingFlowsReverseDiffExt.jl b/ext/NormalizingFlowsReverseDiffExt.jl index 4bd444d1..615f3171 100644 --- a/ext/NormalizingFlowsReverseDiffExt.jl +++ b/ext/NormalizingFlowsReverseDiffExt.jl @@ -1,19 +1,19 @@ module NormalizingFlowsReverseDiffExt if isdefined(Base, :get_extension) - using ReverseDiff - using NormalizingFlows: value_and_gradient! using ADTypes using DiffResults + using NormalizingFlows + using ReverseDiff else - using ..ReverseDiff - using ..NormalizingFlows: value_and_gradient! using ..ADTypes using ..DiffResults + using ..NormalizingFlows + using ..ReverseDiff end # ReverseDiff without compiled tape -function value_and_gradient!( +function NormalizingFlows.value_and_gradient!( ad::ADTypes.AutoReverseDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} tp = ReverseDiff.GradientTape(f, θ) diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl index ab87be68..8f6f1ff7 100644 --- a/ext/NormalizingFlowsZygoteExt.jl +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -1,19 +1,19 @@ module NormalizingFlowsZygoteExt if isdefined(Base, :get_extension) - using Zygote - using NormalizingFlows: value_and_gradient! using ADTypes using DiffResults + using NormalizingFlows + using Zygote else - using ..Zygote - using ..NormalizingFlows: value_and_gradient! using ..ADTypes using ..DiffResults + using ..NormalizingFlows + using ..Zygote end @info "loading zygote ext" -function value_and_gradient!( +function NormalizingFlows.value_and_gradient!( ad::ADTypes.AutoZygote, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} y, back = Zygote.pullback(f, θ) From 25d42113b26b08249bf5c73541f1544b53280a85 Mon Sep 17 00:00:00 2001 From: Xianda Sun Date: Thu, 13 Jul 2023 11:27:34 +0100 Subject: [PATCH 13/40] Applying @devmotion's comment --- ext/NormalizingFlowsEnzymeExt.jl | 7 ++----- ext/NormalizingFlowsForwardDiffExt.jl | 9 +++------ ext/NormalizingFlowsReverseDiffExt.jl | 7 ++----- ext/NormalizingFlowsZygoteExt.jl | 7 ++----- src/NormalizingFlows.jl | 3 +++ 5 files changed, 12 insertions(+), 21 deletions(-) diff --git a/ext/NormalizingFlowsEnzymeExt.jl b/ext/NormalizingFlowsEnzymeExt.jl index 0542c669..4f8d39bd 100644 --- a/ext/NormalizingFlowsEnzymeExt.jl +++ b/ext/NormalizingFlowsEnzymeExt.jl @@ -1,20 +1,17 @@ module NormalizingFlowsEnzymeExt if isdefined(Base, :get_extension) - using ADTypes - using DiffResults using Enzyme using NormalizingFlows + using NormalizingFlows: AutoEnzyme, MutableDiffResult, value!, gradient else - using ..ADTypes - using ..DiffResults using ..Enzyme using ..NormalizingFlows end # Enzyme doesn't support f::Bijectors (see https://github.com/EnzymeAD/Enzyme.jl/issues/916) function NormalizingFlows.value_and_gradient!( - ad::ADTypes.AutoEnzyme, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult + ad::AutoEnzyme, f, θ::AbstractVector{T}, out::MutableDiffResult ) where {T<:Real} y = f(θ) DiffResults.value!(out, y) diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl index c4f70200..a1bf5e6b 100644 --- a/ext/NormalizingFlowsForwardDiffExt.jl +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -1,21 +1,18 @@ module NormalizingFlowsForwardDiffExt if isdefined(Base, :get_extension) - using ADTypes - using DiffResults using ForwardDiff using NormalizingFlows + using NormalizingFlows: AutoForwardDiff, MutableDiffResult else - using ..ADTypes - using ..DiffResults using ..ForwardDiff using ..NormalizingFlows end # extract chunk size from AutoForwardDiff -getchunksize(::ADTypes.AutoForwardDiff{chunksize}) where {chunksize} = chunksize +getchunksize(::AutoForwardDiff{chunksize}) where {chunksize} = chunksize function NormalizingFlows.value_and_gradient!( - ad::ADTypes.AutoForwardDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult + ad::AutoForwardDiff, f, θ::AbstractVector{T}, out::MutableDiffResult ) where {T<:Real} chunk_size = getchunksize(ad) config = if isnothing(chunk_size) diff --git a/ext/NormalizingFlowsReverseDiffExt.jl b/ext/NormalizingFlowsReverseDiffExt.jl index 615f3171..2cecf804 100644 --- a/ext/NormalizingFlowsReverseDiffExt.jl +++ b/ext/NormalizingFlowsReverseDiffExt.jl @@ -1,20 +1,17 @@ module NormalizingFlowsReverseDiffExt if isdefined(Base, :get_extension) - using ADTypes - using DiffResults using NormalizingFlows + using NormalizingFlows: AutoReverseDiff, MutableDiffResult using ReverseDiff else - using ..ADTypes - using ..DiffResults using ..NormalizingFlows using ..ReverseDiff end # ReverseDiff without compiled tape function NormalizingFlows.value_and_gradient!( - ad::ADTypes.AutoReverseDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult + ad::AutoReverseDiff, f, θ::AbstractVector{T}, out::MutableDiffResult ) where {T<:Real} tp = ReverseDiff.GradientTape(f, θ) ReverseDiff.gradient!(out, tp, θ) diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl index 8f6f1ff7..3984f57a 100644 --- a/ext/NormalizingFlowsZygoteExt.jl +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -1,20 +1,17 @@ module NormalizingFlowsZygoteExt if isdefined(Base, :get_extension) - using ADTypes - using DiffResults using NormalizingFlows + using NormalizingFlows: AutoZygote, MutableDiffResult using Zygote else - using ..ADTypes - using ..DiffResults using ..NormalizingFlows using ..Zygote end @info "loading zygote ext" function NormalizingFlows.value_and_gradient!( - ad::ADTypes.AutoZygote, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult + ad::AutoZygote, f, θ::AbstractVector{T}, out::MutableDiffResult ) where {T<:Real} y, back = Zygote.pullback(f, θ) ∇θ = back(one(T)) diff --git a/src/NormalizingFlows.jl b/src/NormalizingFlows.jl index 91457b8e..f5147c8f 100644 --- a/src/NormalizingFlows.jl +++ b/src/NormalizingFlows.jl @@ -10,6 +10,9 @@ using DocStringExtensions export train_flow, elbo, loglikelihood, value_and_gradient! +using ADTypes: AutoEnzyme, AutoForwardDiff, AutoReverseDiff, AutoZygote +using DiffResults: MutableDiffResult + """ train_flow([rng::AbstractRNG, ]vo, flow, args...; kwargs...) From df5edddd303346ee7b587f370d91996ea48b623f Mon Sep 17 00:00:00 2001 From: Xianda Sun Date: Thu, 13 Jul 2023 11:35:08 +0100 Subject: [PATCH 14/40] patch last commit --- ext/NormalizingFlowsEnzymeExt.jl | 4 ++-- ext/NormalizingFlowsZygoteExt.jl | 6 +++--- src/NormalizingFlows.jl | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/NormalizingFlowsEnzymeExt.jl b/ext/NormalizingFlowsEnzymeExt.jl index 4f8d39bd..cbeae581 100644 --- a/ext/NormalizingFlowsEnzymeExt.jl +++ b/ext/NormalizingFlowsEnzymeExt.jl @@ -14,8 +14,8 @@ function NormalizingFlows.value_and_gradient!( ad::AutoEnzyme, f, θ::AbstractVector{T}, out::MutableDiffResult ) where {T<:Real} y = f(θ) - DiffResults.value!(out, y) - ∇θ = DiffResults.gradient(out) + value!(out, y) + ∇θ = gradient(out) fill!(∇θ, zero(T)) Enzyme.autodiff(Enzyme.ReverseWithPrimal, f, Enzyme.Active, Enzyme.Duplicated(θ, ∇θ)) return out diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl index 3984f57a..bf0d29e1 100644 --- a/ext/NormalizingFlowsZygoteExt.jl +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -2,7 +2,7 @@ module NormalizingFlowsZygoteExt if isdefined(Base, :get_extension) using NormalizingFlows - using NormalizingFlows: AutoZygote, MutableDiffResult + using NormalizingFlows: AutoZygote, MutableDiffResult, value!, gradient! using Zygote else using ..NormalizingFlows @@ -15,8 +15,8 @@ function NormalizingFlows.value_and_gradient!( ) where {T<:Real} y, back = Zygote.pullback(f, θ) ∇θ = back(one(T)) - DiffResults.value!(out, y) - DiffResults.gradient!(out, first(∇θ)) + value!(out, y) + gradient!(out, first(∇θ)) return out end diff --git a/src/NormalizingFlows.jl b/src/NormalizingFlows.jl index f5147c8f..977f7d1c 100644 --- a/src/NormalizingFlows.jl +++ b/src/NormalizingFlows.jl @@ -11,7 +11,7 @@ using DocStringExtensions export train_flow, elbo, loglikelihood, value_and_gradient! using ADTypes: AutoEnzyme, AutoForwardDiff, AutoReverseDiff, AutoZygote -using DiffResults: MutableDiffResult +using DiffResults: MutableDiffResult, value!, gradient, gradient! """ train_flow([rng::AbstractRNG, ]vo, flow, args...; kwargs...) From d44143b4f45505923a5c315129c782f650226af5 Mon Sep 17 00:00:00 2001 From: Xianda Sun Date: Thu, 13 Jul 2023 11:49:35 +0100 Subject: [PATCH 15/40] patch for julia 1.6 --- ext/NormalizingFlowsEnzymeExt.jl | 1 + ext/NormalizingFlowsForwardDiffExt.jl | 1 + ext/NormalizingFlowsReverseDiffExt.jl | 1 + ext/NormalizingFlowsZygoteExt.jl | 1 + 4 files changed, 4 insertions(+) diff --git a/ext/NormalizingFlowsEnzymeExt.jl b/ext/NormalizingFlowsEnzymeExt.jl index cbeae581..4e80dfa7 100644 --- a/ext/NormalizingFlowsEnzymeExt.jl +++ b/ext/NormalizingFlowsEnzymeExt.jl @@ -7,6 +7,7 @@ if isdefined(Base, :get_extension) else using ..Enzyme using ..NormalizingFlows + using ..NormalizingFlows: AutoEnzyme, MutableDiffResult, value!, gradient end # Enzyme doesn't support f::Bijectors (see https://github.com/EnzymeAD/Enzyme.jl/issues/916) diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl index a1bf5e6b..58a94c8d 100644 --- a/ext/NormalizingFlowsForwardDiffExt.jl +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -7,6 +7,7 @@ if isdefined(Base, :get_extension) else using ..ForwardDiff using ..NormalizingFlows + using ..NormalizingFlows: AutoForwardDiff, MutableDiffResult end # extract chunk size from AutoForwardDiff diff --git a/ext/NormalizingFlowsReverseDiffExt.jl b/ext/NormalizingFlowsReverseDiffExt.jl index 2cecf804..0233972f 100644 --- a/ext/NormalizingFlowsReverseDiffExt.jl +++ b/ext/NormalizingFlowsReverseDiffExt.jl @@ -6,6 +6,7 @@ if isdefined(Base, :get_extension) using ReverseDiff else using ..NormalizingFlows + using ..NormalizingFlows: AutoReverseDiff, MutableDiffResult using ..ReverseDiff end diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl index bf0d29e1..77245e67 100644 --- a/ext/NormalizingFlowsZygoteExt.jl +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -6,6 +6,7 @@ if isdefined(Base, :get_extension) using Zygote else using ..NormalizingFlows + using ..NormalizingFlows: AutoZygote, MutableDiffResult, value!, gradient! using ..Zygote end From b03e9221c3a3423d1486ea3f4426c7bd6c7d6173 Mon Sep 17 00:00:00 2001 From: zuheng Date: Thu, 13 Jul 2023 08:02:44 -0700 Subject: [PATCH 16/40] loading dep pkgs from main pkg instead of functions for explicitness --- ext/NormalizingFlowsEnzymeExt.jl | 10 +++++----- ext/NormalizingFlowsForwardDiffExt.jl | 6 +++--- ext/NormalizingFlowsReverseDiffExt.jl | 6 +++--- ext/NormalizingFlowsZygoteExt.jl | 10 +++++----- src/NormalizingFlows.jl | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ext/NormalizingFlowsEnzymeExt.jl b/ext/NormalizingFlowsEnzymeExt.jl index 4e80dfa7..1b59cad8 100644 --- a/ext/NormalizingFlowsEnzymeExt.jl +++ b/ext/NormalizingFlowsEnzymeExt.jl @@ -3,20 +3,20 @@ module NormalizingFlowsEnzymeExt if isdefined(Base, :get_extension) using Enzyme using NormalizingFlows - using NormalizingFlows: AutoEnzyme, MutableDiffResult, value!, gradient + using NormalizingFlows: ADTypes, DiffResults else using ..Enzyme using ..NormalizingFlows - using ..NormalizingFlows: AutoEnzyme, MutableDiffResult, value!, gradient + using ..NormalizingFlows: ADTypes, DiffResults end # Enzyme doesn't support f::Bijectors (see https://github.com/EnzymeAD/Enzyme.jl/issues/916) function NormalizingFlows.value_and_gradient!( - ad::AutoEnzyme, f, θ::AbstractVector{T}, out::MutableDiffResult + ad::ADTypes.AutoEnzyme, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} y = f(θ) - value!(out, y) - ∇θ = gradient(out) + DiffResults.value!(out, y) + ∇θ = DiffResults.gradient(out) fill!(∇θ, zero(T)) Enzyme.autodiff(Enzyme.ReverseWithPrimal, f, Enzyme.Active, Enzyme.Duplicated(θ, ∇θ)) return out diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl index 58a94c8d..937d099d 100644 --- a/ext/NormalizingFlowsForwardDiffExt.jl +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -3,17 +3,17 @@ module NormalizingFlowsForwardDiffExt if isdefined(Base, :get_extension) using ForwardDiff using NormalizingFlows - using NormalizingFlows: AutoForwardDiff, MutableDiffResult + using NormalizingFlows: ADTypes, DiffResults else using ..ForwardDiff using ..NormalizingFlows - using ..NormalizingFlows: AutoForwardDiff, MutableDiffResult + using ..NormalizingFlows: ADTypes, DiffResults end # extract chunk size from AutoForwardDiff getchunksize(::AutoForwardDiff{chunksize}) where {chunksize} = chunksize function NormalizingFlows.value_and_gradient!( - ad::AutoForwardDiff, f, θ::AbstractVector{T}, out::MutableDiffResult + ad::ADTypes.AutoForwardDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} chunk_size = getchunksize(ad) config = if isnothing(chunk_size) diff --git a/ext/NormalizingFlowsReverseDiffExt.jl b/ext/NormalizingFlowsReverseDiffExt.jl index 0233972f..1bd39dc4 100644 --- a/ext/NormalizingFlowsReverseDiffExt.jl +++ b/ext/NormalizingFlowsReverseDiffExt.jl @@ -2,17 +2,17 @@ module NormalizingFlowsReverseDiffExt if isdefined(Base, :get_extension) using NormalizingFlows - using NormalizingFlows: AutoReverseDiff, MutableDiffResult + using NormalizingFlows: ADTypes, DiffResults using ReverseDiff else using ..NormalizingFlows - using ..NormalizingFlows: AutoReverseDiff, MutableDiffResult + using ..NormalizingFlows: ADTypes, DiffResults using ..ReverseDiff end # ReverseDiff without compiled tape function NormalizingFlows.value_and_gradient!( - ad::AutoReverseDiff, f, θ::AbstractVector{T}, out::MutableDiffResult + ad::ADTypes.AutoReverseDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} tp = ReverseDiff.GradientTape(f, θ) ReverseDiff.gradient!(out, tp, θ) diff --git a/ext/NormalizingFlowsZygoteExt.jl b/ext/NormalizingFlowsZygoteExt.jl index 77245e67..7d7d4b63 100644 --- a/ext/NormalizingFlowsZygoteExt.jl +++ b/ext/NormalizingFlowsZygoteExt.jl @@ -2,22 +2,22 @@ module NormalizingFlowsZygoteExt if isdefined(Base, :get_extension) using NormalizingFlows - using NormalizingFlows: AutoZygote, MutableDiffResult, value!, gradient! + using NormalizingFlows: ADTypes, DiffResults using Zygote else using ..NormalizingFlows - using ..NormalizingFlows: AutoZygote, MutableDiffResult, value!, gradient! + using ..NormalizingFlows: ADTypes, DiffResults using ..Zygote end @info "loading zygote ext" function NormalizingFlows.value_and_gradient!( - ad::AutoZygote, f, θ::AbstractVector{T}, out::MutableDiffResult + ad::ADTypes.AutoZygote, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} y, back = Zygote.pullback(f, θ) ∇θ = back(one(T)) - value!(out, y) - gradient!(out, first(∇θ)) + DiffResults.value!(out, y) + DiffResults.gradient!(out, first(∇θ)) return out end diff --git a/src/NormalizingFlows.jl b/src/NormalizingFlows.jl index 977f7d1c..2d708929 100644 --- a/src/NormalizingFlows.jl +++ b/src/NormalizingFlows.jl @@ -10,8 +10,8 @@ using DocStringExtensions export train_flow, elbo, loglikelihood, value_and_gradient! -using ADTypes: AutoEnzyme, AutoForwardDiff, AutoReverseDiff, AutoZygote -using DiffResults: MutableDiffResult, value!, gradient, gradient! +using ADTypes +using DiffResults """ train_flow([rng::AbstractRNG, ]vo, flow, args...; kwargs...) From e9acf70165eb43d0e4393f046cd3a3eff1fda5b3 Mon Sep 17 00:00:00 2001 From: zuheng Date: Thu, 13 Jul 2023 09:12:18 -0700 Subject: [PATCH 17/40] fixing test err --- example/neural_spline_flow/example.jl | 22 ++++++++++++++++++++++ ext/NormalizingFlowsForwardDiffExt.jl | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 example/neural_spline_flow/example.jl diff --git a/example/neural_spline_flow/example.jl b/example/neural_spline_flow/example.jl new file mode 100644 index 00000000..d54fce71 --- /dev/null +++ b/example/neural_spline_flow/example.jl @@ -0,0 +1,22 @@ +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using FunctionChains +using NormalizingFlows +using Flux: f32 +using Plots +include("../common.jl") + +Random.seed!(123) +rng = Random.default_rng() + +###################################### +# 2d Banana as the target distribution +###################################### +include("../targets/banana.jl") + +# create target p +p = Banana(2, 1.0f-1, 100.0f0) +logp = Base.Fix1(logpdf, p) + +# work in progress, not for review \ No newline at end of file diff --git a/ext/NormalizingFlowsForwardDiffExt.jl b/ext/NormalizingFlowsForwardDiffExt.jl index 937d099d..500d54f4 100644 --- a/ext/NormalizingFlowsForwardDiffExt.jl +++ b/ext/NormalizingFlowsForwardDiffExt.jl @@ -11,7 +11,7 @@ else end # extract chunk size from AutoForwardDiff -getchunksize(::AutoForwardDiff{chunksize}) where {chunksize} = chunksize +getchunksize(::ADTypes.AutoForwardDiff{chunksize}) where {chunksize} = chunksize function NormalizingFlows.value_and_gradient!( ad::ADTypes.AutoForwardDiff, f, θ::AbstractVector{T}, out::DiffResults.MutableDiffResult ) where {T<:Real} From c6bf68bdfc8d30e96482d9689dc8fab97a9c6b3f Mon Sep 17 00:00:00 2001 From: zuheng Date: Thu, 13 Jul 2023 09:12:46 -0700 Subject: [PATCH 18/40] rm unready examples --- example/neural_spline_flow/example.jl | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 example/neural_spline_flow/example.jl diff --git a/example/neural_spline_flow/example.jl b/example/neural_spline_flow/example.jl deleted file mode 100644 index d54fce71..00000000 --- a/example/neural_spline_flow/example.jl +++ /dev/null @@ -1,22 +0,0 @@ -using Random, Distributions, LinearAlgebra, Bijectors -using ADTypes -using Optimisers -using FunctionChains -using NormalizingFlows -using Flux: f32 -using Plots -include("../common.jl") - -Random.seed!(123) -rng = Random.default_rng() - -###################################### -# 2d Banana as the target distribution -###################################### -include("../targets/banana.jl") - -# create target p -p = Banana(2, 1.0f-1, 100.0f0) -logp = Base.Fix1(logpdf, p) - -# work in progress, not for review \ No newline at end of file From fabf20a55a2341e5eb7657a8746aae41c42f2906 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Sun, 23 Jul 2023 20:58:30 -0700 Subject: [PATCH 19/40] update realnvp --- example/RealNVP/AffineCoupling.jl | 108 ++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 example/RealNVP/AffineCoupling.jl diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl new file mode 100644 index 00000000..ff34ed3e --- /dev/null +++ b/example/RealNVP/AffineCoupling.jl @@ -0,0 +1,108 @@ +using Flux +using Functors +using Bijectors +using Bijectors:∘, partition, combine, PartitionMask +import Bijectors:transform,with_logabsdet_jacobian,logabsdetjac + + +""" +Affinecoupling layer for RealNVP "(http://proceedings.mlr.press/v118/fjelde20a/fjelde20a.pdf)" +""" +struct AffineCoupling <: Bijectors.Bijector + D::Int + Mask::Bijectors.PartitionMask + s::Flux.Chain + t::Flux.Chain +end + +# let params track field s and t +@functor AffineCoupling (s, t) + +function AffineCoupling( + D::Int, # dimension of input + hdims::Int, # dimension of hidden units for s and t + mask_idx::AbstractVector # index of dimensione that one wants to apply transformations on + ) + cdims = D ÷ 2 # D is even (and D ≥ 4 since PartitionMask requires D ≥ 3) + s = Chain(Flux.Dense(cdims, hdims, leakyrelu), Flux.Dense(hdims, hdims, leakyrelu), Flux.Dense(hdims, cdims)) + t = Chain(Flux.Dense(cdims, hdims, leakyrelu), Flux.Dense(hdims, hdims, leakyrelu), Flux.Dense(hdims, cdims)) + Mask = Bijectors.PartitionMask(D, mask_idx) + AffineCoupling(D, Mask, s, t) +end + +function Bijectors.transform(af::AffineCoupling, x::AbstractVector) + # partition vector using 'af.Mask::PartitionMask` + x₁, x₂, x₃= Bijectors.partition(af.Mask, x) + y₁ = x₁ .* exp.(af.s(x₂)) .+ af.t(x₂) + return Bijectors.combine(af.Mask, y₁, x₂, x₃) +end + +function (af::AffineCoupling)(x::AbstractArray) + return Bijectors.transform(af, x) +end + +function Bijectors.with_logabsdet_jacobian(af::AffineCoupling, x::AbstractVector) + x_1, x_2, x_3 = Bijectors.partition(af.Mask, x) + y_1 = exp.(af.s(x_2)) .* x_1 .+ af.t(x_2) + logjac = sum(af.s(x_2)) # logabsdetjac of exp is simply the argument + return combine(af.Mask, y_1, x_2, x_3), logjac +end + + +function Bijectors.transform(iaf::Inverse{<:AffineCoupling}, y::AbstractVector) + af = iaf.orig + # partition vector using `af.mask::PartitionMask` + y_1, y_2, y_3 = partition(af.Mask, y) + # inverse transformation + x_1 = (y_1 .- af.t(y_2)) .* exp.(-af.s(y_2)) + return combine(af.Mask, x_1, y_2, y_3) +end + +function Bijectors.with_logabsdet_jacobian(iaf::Inverse{<:AffineCoupling}, y::AbstractVector) + af = iaf.orig + # partition vector using `af.mask::PartitionMask` + y_1, y_2, y_3 = partition(af.Mask, y) + # inverse transformation + x_1 = (y_1 .- af.t(y_2)) .* exp.(-af.s(y_2)) + logjac = -sum(af.s(y_2)) + return combine(af.Mask, x_1, y_2, y_3), logjac +end + + +function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) + x_1, x_2, x_3 = partition(af.Mask, x) + logjac = sum(af.s(x_2)) + return logjac +end +# Todo: specialized method for 2d input +# `partition` does support 2d input + + + +# # test invertibility +# Ls = [ +# AffineCoupling(4, 8, 1:2), +# AffineCoupling(4, 8, 3:4), +# AffineCoupling(4, 8, 1:2), +# AffineCoupling(4, 8, 3:4), +# ] +# T = Bijectors.∘(Ls...) +# B = inverse(T) + +# x0 = randn(4) +# x00 = B(T(x0)) +# norm(x0 .- x00) + +# # test autograd +# Flux.params(T) +# function test(x) +# sum(abs2,T(x)) +# end + +# using Zygote +# Zygote.gradient(test, x0) + + +# X = randn(2) +# mask = Bijectors.PartitionMask(2, 1:1) +# x1, x2, x3 = Bijectors.partition(mask, X) \ No newline at end of file From 7964f917cb85ab548c3a855d3473f25291c0e188 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Sun, 23 Jul 2023 22:02:28 -0700 Subject: [PATCH 20/40] minor ed --- example/RealNVP/AffineCoupling.jl | 36 ++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl index ff34ed3e..4383d3b0 100644 --- a/example/RealNVP/AffineCoupling.jl +++ b/example/RealNVP/AffineCoupling.jl @@ -1,9 +1,8 @@ using Flux using Functors using Bijectors -using Bijectors:∘, partition, combine, PartitionMask -import Bijectors:transform,with_logabsdet_jacobian,logabsdetjac - +using Bijectors: ∘, partition, combine, PartitionMask +import Bijectors: transform, with_logabsdet_jacobian, logabsdetjac """ Affinecoupling layer for RealNVP "(http://proceedings.mlr.press/v118/fjelde20a/fjelde20a.pdf)" @@ -21,18 +20,26 @@ end function AffineCoupling( D::Int, # dimension of input hdims::Int, # dimension of hidden units for s and t - mask_idx::AbstractVector # index of dimensione that one wants to apply transformations on - ) + mask_idx::AbstractVector, # index of dimensione that one wants to apply transformations on +) cdims = D ÷ 2 # D is even (and D ≥ 4 since PartitionMask requires D ≥ 3) - s = Chain(Flux.Dense(cdims, hdims, leakyrelu), Flux.Dense(hdims, hdims, leakyrelu), Flux.Dense(hdims, cdims)) - t = Chain(Flux.Dense(cdims, hdims, leakyrelu), Flux.Dense(hdims, hdims, leakyrelu), Flux.Dense(hdims, cdims)) + s = Chain( + Flux.Dense(cdims, hdims, leakyrelu), + Flux.Dense(hdims, hdims, leakyrelu), + Flux.Dense(hdims, cdims), + ) + t = Chain( + Flux.Dense(cdims, hdims, leakyrelu), + Flux.Dense(hdims, hdims, leakyrelu), + Flux.Dense(hdims, cdims), + ) Mask = Bijectors.PartitionMask(D, mask_idx) - AffineCoupling(D, Mask, s, t) + return AffineCoupling(D, Mask, s, t) end function Bijectors.transform(af::AffineCoupling, x::AbstractVector) # partition vector using 'af.Mask::PartitionMask` - x₁, x₂, x₃= Bijectors.partition(af.Mask, x) + x₁, x₂, x₃ = Bijectors.partition(af.Mask, x) y₁ = x₁ .* exp.(af.s(x₂)) .+ af.t(x₂) return Bijectors.combine(af.Mask, y₁, x₂, x₃) end @@ -48,7 +55,6 @@ function Bijectors.with_logabsdet_jacobian(af::AffineCoupling, x::AbstractVector return combine(af.Mask, y_1, x_2, x_3), logjac end - function Bijectors.transform(iaf::Inverse{<:AffineCoupling}, y::AbstractVector) af = iaf.orig # partition vector using `af.mask::PartitionMask` @@ -58,7 +64,9 @@ function Bijectors.transform(iaf::Inverse{<:AffineCoupling}, y::AbstractVector) return combine(af.Mask, x_1, y_2, y_3) end -function Bijectors.with_logabsdet_jacobian(iaf::Inverse{<:AffineCoupling}, y::AbstractVector) +function Bijectors.with_logabsdet_jacobian( + iaf::Inverse{<:AffineCoupling}, y::AbstractVector +) af = iaf.orig # partition vector using `af.mask::PartitionMask` y_1, y_2, y_3 = partition(af.Mask, y) @@ -68,16 +76,11 @@ function Bijectors.with_logabsdet_jacobian(iaf::Inverse{<:AffineCoupling}, y::Ab return combine(af.Mask, x_1, y_2, y_3), logjac end - function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) x_1, x_2, x_3 = partition(af.Mask, x) logjac = sum(af.s(x_2)) return logjac end -# Todo: specialized method for 2d input -# `partition` does support 2d input - - # # test invertibility # Ls = [ @@ -102,7 +105,6 @@ end # using Zygote # Zygote.gradient(test, x0) - # X = randn(2) # mask = Bijectors.PartitionMask(2, 1:1) # x1, x2, x3 = Bijectors.partition(mask, X) \ No newline at end of file From 01de9d4ee82746f5e419929464199532a7fd6a8d Mon Sep 17 00:00:00 2001 From: Zuheng Date: Sun, 23 Jul 2023 22:38:25 -0700 Subject: [PATCH 21/40] removing unnecessary import --- example/RealNVP/AffineCoupling.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl index 4383d3b0..79294757 100644 --- a/example/RealNVP/AffineCoupling.jl +++ b/example/RealNVP/AffineCoupling.jl @@ -1,8 +1,7 @@ using Flux using Functors using Bijectors -using Bijectors: ∘, partition, combine, PartitionMask -import Bijectors: transform, with_logabsdet_jacobian, logabsdetjac +using Bijectors: partition, combine, PartitionMask """ Affinecoupling layer for RealNVP "(http://proceedings.mlr.press/v118/fjelde20a/fjelde20a.pdf)" From 6993d339f1ec76bca160c8c4bec56e906266a2c7 Mon Sep 17 00:00:00 2001 From: zuhengxu Date: Sun, 30 Jul 2023 18:00:11 -0700 Subject: [PATCH 22/40] refactor affinecoupling and example/ --- example/Hamiltonian/hamflow.jl | 0 example/RealNVP/AffineCoupling.jl | 55 ++++++++++------------ example/neural_spline_flow/nsf_layer.jl | 13 ++---- example/util.jl | 9 ++++ src/util.jl | 61 +++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 example/Hamiltonian/hamflow.jl create mode 100644 example/util.jl create mode 100644 src/util.jl diff --git a/example/Hamiltonian/hamflow.jl b/example/Hamiltonian/hamflow.jl new file mode 100644 index 00000000..e69de29b diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl index 79294757..e395c97e 100644 --- a/example/RealNVP/AffineCoupling.jl +++ b/example/RealNVP/AffineCoupling.jl @@ -3,12 +3,14 @@ using Functors using Bijectors using Bijectors: partition, combine, PartitionMask +include("../util.jl") + """ Affinecoupling layer for RealNVP "(http://proceedings.mlr.press/v118/fjelde20a/fjelde20a.pdf)" """ struct AffineCoupling <: Bijectors.Bijector - D::Int - Mask::Bijectors.PartitionMask + dim::Int + mask::Bijectors.PartitionMask s::Flux.Chain t::Flux.Chain end @@ -17,29 +19,29 @@ end @functor AffineCoupling (s, t) function AffineCoupling( - D::Int, # dimension of input + dim::Int, # dimension of input hdims::Int, # dimension of hidden units for s and t mask_idx::AbstractVector, # index of dimensione that one wants to apply transformations on ) - cdims = D ÷ 2 # D is even (and D ≥ 4 since PartitionMask requires D ≥ 3) - s = Chain( - Flux.Dense(cdims, hdims, leakyrelu), - Flux.Dense(hdims, hdims, leakyrelu), - Flux.Dense(hdims, cdims), - ) - t = Chain( - Flux.Dense(cdims, hdims, leakyrelu), - Flux.Dense(hdims, hdims, leakyrelu), - Flux.Dense(hdims, cdims), - ) - Mask = Bijectors.PartitionMask(D, mask_idx) - return AffineCoupling(D, Mask, s, t) + cdims = length(mask_idx) # dimension of parts used to construct coupling law + s = MLP_3layer(cdims, hdims, cdims) + t = MLP_3layer(cdims, hdims, cdims) + mask = Bijectors.PartitionMask(dim, mask_idx) + return AffineCoupling(dim, mask, s, t) end +## scaling parameterize using exp +# function Bijectors.transform(af::AffineCoupling, x::AbstractVector) +# # partition vector using 'af.mask::PartitionMask` +# x₁, x₂, x₃ = Bijectors.partition(af.Mask, x) +# y₁ = x₁ .* exp.(af.s(x₂)) .+ af.t(x₂) +# return Bijectors.combine(af.mask, y₁, x₂, x₃) +# end + function Bijectors.transform(af::AffineCoupling, x::AbstractVector) - # partition vector using 'af.Mask::PartitionMask` + # partition vector using 'af.mask::PartitionMask` x₁, x₂, x₃ = Bijectors.partition(af.Mask, x) - y₁ = x₁ .* exp.(af.s(x₂)) .+ af.t(x₂) + y₁ = x₁ .* af.s(x₂) .+ af.t(x₂) return Bijectors.combine(af.Mask, y₁, x₂, x₃) end @@ -48,19 +50,10 @@ function (af::AffineCoupling)(x::AbstractArray) end function Bijectors.with_logabsdet_jacobian(af::AffineCoupling, x::AbstractVector) - x_1, x_2, x_3 = Bijectors.partition(af.Mask, x) + x_1, x_2, x_3 = Bijectors.partition(af.mask, x) y_1 = exp.(af.s(x_2)) .* x_1 .+ af.t(x_2) - logjac = sum(af.s(x_2)) # logabsdetjac of exp is simply the argument - return combine(af.Mask, y_1, x_2, x_3), logjac -end - -function Bijectors.transform(iaf::Inverse{<:AffineCoupling}, y::AbstractVector) - af = iaf.orig - # partition vector using `af.mask::PartitionMask` - y_1, y_2, y_3 = partition(af.Mask, y) - # inverse transformation - x_1 = (y_1 .- af.t(y_2)) .* exp.(-af.s(y_2)) - return combine(af.Mask, x_1, y_2, y_3) + logjac = sum(log ∘ abs, af.s(x_2)) # logabsdetjac of exp is simply the argument + return combine(af.mask, y_1, x_2, x_3), logjac end function Bijectors.with_logabsdet_jacobian( @@ -82,7 +75,7 @@ function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) end # # test invertibility -# Ls = [ +# Ls = [ # AffineCoupling(4, 8, 1:2), # AffineCoupling(4, 8, 3:4), # AffineCoupling(4, 8, 1:2), diff --git a/example/neural_spline_flow/nsf_layer.jl b/example/neural_spline_flow/nsf_layer.jl index 6f3b2a7f..5cabea1f 100644 --- a/example/neural_spline_flow/nsf_layer.jl +++ b/example/neural_spline_flow/nsf_layer.jl @@ -3,6 +3,7 @@ using Functors using Bijectors using Bijectors: partition, PartitionMask +include("../util.jl") """ Neural Rational quadratic Spline layer @@ -18,14 +19,6 @@ struct NeuralSplineLayer{T1,T2,A<:AbstractVecOrMat{T1}} <: Bijectors.Bijector B::T2 # bound of the knots end -function MLP_3layer(input_dim::Int, hdims::Int, output_dim::Int; activation=Flux.leakyrelu) - return Chain( - Flux.Dense(input_dim, hdims, activation), - Flux.Dense(hdims, hdims, activation), - Flux.Dense(hdims, output_dim), - ) -end - function NeuralSplineLayer( dim::Int, # dimension of input hdims::Int, # dimension of hidden units for s and t @@ -82,8 +75,8 @@ function Bijectors.logabsdetjac( nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector ) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) - Rqs = instantiate_rqs(nsl, x_2) - logjac = logabsdetjac(Rqs, x_1) + rqs = instantiate_rqs(nsl, x_2) + logjac = logabsdetjac(rqs, x_1) return logjac end diff --git a/example/util.jl b/example/util.jl new file mode 100644 index 00000000..d98bff82 --- /dev/null +++ b/example/util.jl @@ -0,0 +1,9 @@ +using Flux + +function MLP_3layer(input_dim::Int, hdims::Int, output_dim::Int; activation=Flux.leakyrelu) + return Chain( + Flux.Dense(input_dim, hdims, activation), + Flux.Dense(hdims, hdims, activation), + Flux.Dense(hdims, output_dim), + ) +end \ No newline at end of file diff --git a/src/util.jl b/src/util.jl new file mode 100644 index 00000000..e63c5895 --- /dev/null +++ b/src/util.jl @@ -0,0 +1,61 @@ +using Functors +using Bijectors, Flux + +# falttening and unflattening approach from: https://github.com/TuringLang/AdvancedVI.jl/discussions/46#discussioncomment-5543111 +inner_flatten(x::Real) = [x], only +inner_flatten(x::AbstractVector{<:Real}) = x, identity +inner_flatten(x::AbstractArray{<:Real}) = vec(x), Base.Fix2(reshape, size(x)) + +function inner_flatten(original::Tuple) + vecs_and_unflattens = map(inner_flatten, original) + vecs = map(first, vecs_and_unflattens) + unflattens = map(last, vecs_and_unflattens) + + lengths = map(length, vecs) + end_indices = cumsum(lengths) + + N = length(original) + + function unflatten_Tuple(x) + ntuple(N) do i + v = original[i] + unflatten = unflattens[i] + l = lengths[i] + end_idx = end_indices[i] + start_idx = end_idx - l + 1 + return unflatten(@view(x[start_idx:end_idx])) + end + end + + return reduce(vcat, vecs), unflatten_Tuple +end + +function inner_flatten(x::NamedTuple{names}) where {names} + vec, re = inner_flatten(values(x)) + unflatten_NamedTuple(x) = NamedTuple{names}(re(x)) + return vec, unflatten_NamedTuple +end + +""" + flatten(x) + +Return a the flattened vector of parameters and a function to reconstruct `x` from the vector. + +This uses `Functors.functor` to get the parameters of `x` and `inner_flatten` to flatten them. +""" +function flatten(x) + params, re = Functors.functor(x) + params_vec, re_vec = inner_flatten(params) + return params_vec, re ∘ re_vec +end + +# Functors for Composed type +function Functors.functor(::Type{<:ComposedFunction}, f) + outer, re_outer = Functors.functor(f.outer) + inner, re_inner = Functors.functor(f.inner) + function reconstruct_ComposedFunction(x) + return ComposedFunction(re_outer(x.outer), re_inner(x.inner)) + end + + return (; outer, inner), reconstruct_ComposedFunction +end From 810881f6e4b8241cc1eab71fc5fe6ba26fe0ab04 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Sun, 30 Jul 2023 21:44:55 -0700 Subject: [PATCH 23/40] debug affinecoupling flow --- example/RealNVP/AffineCoupling.jl | 49 ++++++------------------- example/neural_spline_flow/nsf_layer.jl | 6 +-- 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl index e395c97e..533c2ae5 100644 --- a/example/RealNVP/AffineCoupling.jl +++ b/example/RealNVP/AffineCoupling.jl @@ -26,7 +26,7 @@ function AffineCoupling( cdims = length(mask_idx) # dimension of parts used to construct coupling law s = MLP_3layer(cdims, hdims, cdims) t = MLP_3layer(cdims, hdims, cdims) - mask = Bijectors.PartitionMask(dim, mask_idx) + mask = PartitionMask(dim, mask_idx) return AffineCoupling(dim, mask, s, t) end @@ -40,19 +40,19 @@ end function Bijectors.transform(af::AffineCoupling, x::AbstractVector) # partition vector using 'af.mask::PartitionMask` - x₁, x₂, x₃ = Bijectors.partition(af.Mask, x) + x₁, x₂, x₃ = partition(af.mask, x) y₁ = x₁ .* af.s(x₂) .+ af.t(x₂) - return Bijectors.combine(af.Mask, y₁, x₂, x₃) + return combine(af.mask, y₁, x₂, x₃) end function (af::AffineCoupling)(x::AbstractArray) - return Bijectors.transform(af, x) + return transform(af, x) end function Bijectors.with_logabsdet_jacobian(af::AffineCoupling, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(af.mask, x) y_1 = exp.(af.s(x_2)) .* x_1 .+ af.t(x_2) - logjac = sum(log ∘ abs, af.s(x_2)) # logabsdetjac of exp is simply the argument + logjac = sum(log ∘ abs, af.s(x_2)) return combine(af.mask, y_1, x_2, x_3), logjac end @@ -61,42 +61,15 @@ function Bijectors.with_logabsdet_jacobian( ) af = iaf.orig # partition vector using `af.mask::PartitionMask` - y_1, y_2, y_3 = partition(af.Mask, y) + y_1, y_2, y_3 = partition(af.mask, y) # inverse transformation - x_1 = (y_1 .- af.t(y_2)) .* exp.(-af.s(y_2)) - logjac = -sum(af.s(y_2)) - return combine(af.Mask, x_1, y_2, y_3), logjac + x_1 = (y_1 .- af.t(y_2)) ./ af.s(y_2) + logjac = -sum(log ∘ abs, af.s(y_2)) + return combine(af.mask, x_1, y_2, y_3), logjac end function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) - x_1, x_2, x_3 = partition(af.Mask, x) - logjac = sum(af.s(x_2)) + x_1, x_2, x_3 = partition(af.mask, x) + logjac = sum(log ∘ abs, af.s(x_2)) return logjac end - -# # test invertibility -# Ls = [ -# AffineCoupling(4, 8, 1:2), -# AffineCoupling(4, 8, 3:4), -# AffineCoupling(4, 8, 1:2), -# AffineCoupling(4, 8, 3:4), -# ] -# T = Bijectors.∘(Ls...) -# B = inverse(T) - -# x0 = randn(4) -# x00 = B(T(x0)) -# norm(x0 .- x00) - -# # test autograd -# Flux.params(T) -# function test(x) -# sum(abs2,T(x)) -# end - -# using Zygote -# Zygote.gradient(test, x0) - -# X = randn(2) -# mask = Bijectors.PartitionMask(2, 1:1) -# x1, x2, x3 = Bijectors.partition(mask, X) \ No newline at end of file diff --git a/example/neural_spline_flow/nsf_layer.jl b/example/neural_spline_flow/nsf_layer.jl index 5cabea1f..6820f48f 100644 --- a/example/neural_spline_flow/nsf_layer.jl +++ b/example/neural_spline_flow/nsf_layer.jl @@ -31,8 +31,8 @@ function NeuralSplineLayer( w = [MLP_3layer(input_dims, hdims, K) for i in 1:num_of_transformed_dims] h = [MLP_3layer(input_dims, hdims, K) for i in 1:num_of_transformed_dims] d = [MLP_3layer(input_dims, hdims, K - 1) for i in 1:num_of_transformed_dims] - mask = Bijectors.PartitionMask(D, mask_idx) - return NeuralSplineLayer(D, mask, w, h, d, B) + mask = Bijectors.PartitionMask(dim, mask_idx) + return NeuralSplineLayer(dim, mask, w, h, d, B) end @functor NeuralSplineLayer (w, h, d) @@ -63,7 +63,7 @@ function Bijectors.transform( y1, y2, y3 = partition(nsl.mask, y) rqs = instantiate_rqs(nsl, y2) x1 = transform(Inverse(rqs), y1) - return combine(nsl.mask, x1, y2, y3) + return Bijectors.combine(nsl.mask, x1, y2, y3) end function (nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}})(x::AbstractVector) From 0277c9e91cc548e8ed629468f2ba9160ea02820d Mon Sep 17 00:00:00 2001 From: Zuheng Date: Sun, 30 Jul 2023 22:01:58 -0700 Subject: [PATCH 24/40] adapt to the updated autoforwarddiff to resolve test err --- test/ad.jl | 2 +- test/interface.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ad.jl b/test/ad.jl index 32534048..ac3a9746 100644 --- a/test/ad.jl +++ b/test/ad.jl @@ -7,7 +7,7 @@ @testset "$at" for at in [ ADTypes.AutoZygote(), - ADTypes.AutoForwardDiff(chuncksize), + ADTypes.AutoForwardDiff(; chuncksize=chuncksize), ADTypes.AutoForwardDiff(), ADTypes.AutoReverseDiff(false), ADTypes.AutoEnzyme(), diff --git a/test/interface.jl b/test/interface.jl index ef47690c..114501df 100644 --- a/test/interface.jl +++ b/test/interface.jl @@ -2,7 +2,7 @@ chuncksize = 4 @testset "$adtype" for adtype in [ ADTypes.AutoZygote(), - ADTypes.AutoForwardDiff(chuncksize), + ADTypes.AutoForwardDiff(; chunksize=chuncksize), ADTypes.AutoForwardDiff(), ADTypes.AutoReverseDiff(false), # ADTypes.AutoEnzyme(), # doesn't work for Enzyme From 4bc02bf318ac8ae6089cea943d3186cce5fa547d Mon Sep 17 00:00:00 2001 From: Zuheng Date: Sun, 30 Jul 2023 23:31:02 -0700 Subject: [PATCH 25/40] fix test err --- test/ad.jl | 4 ++-- test/interface.jl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ad.jl b/test/ad.jl index ac3a9746..a394d806 100644 --- a/test/ad.jl +++ b/test/ad.jl @@ -3,11 +3,11 @@ @testset "$T" for T in [Float32, Float64] x = randn(T, 10) - chuncksize = size(x, 1) + chunksize = size(x, 1) @testset "$at" for at in [ ADTypes.AutoZygote(), - ADTypes.AutoForwardDiff(; chuncksize=chuncksize), + ADTypes.AutoForwardDiff(; chunksize=chunksize), ADTypes.AutoForwardDiff(), ADTypes.AutoReverseDiff(false), ADTypes.AutoEnzyme(), diff --git a/test/interface.jl b/test/interface.jl index 114501df..a3540979 100644 --- a/test/interface.jl +++ b/test/interface.jl @@ -1,8 +1,8 @@ @testset "learining 2d Gaussian" begin - chuncksize = 4 + chunksize = 4 @testset "$adtype" for adtype in [ ADTypes.AutoZygote(), - ADTypes.AutoForwardDiff(; chunksize=chuncksize), + ADTypes.AutoForwardDiff(; chunksize=chunksize), ADTypes.AutoForwardDiff(), ADTypes.AutoReverseDiff(false), # ADTypes.AutoEnzyme(), # doesn't work for Enzyme From 144c668150b1f9f9bb641b8f2eddcf675ce9cd33 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Mon, 31 Jul 2023 12:42:11 -0700 Subject: [PATCH 26/40] add new implementation of affcoupling using Bijectors.Coupling --- example/RealNVP/AffineCoupling.jl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl index 533c2ae5..50e75220 100644 --- a/example/RealNVP/AffineCoupling.jl +++ b/example/RealNVP/AffineCoupling.jl @@ -73,3 +73,34 @@ function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) logjac = sum(log ∘ abs, af.s(x_2)) return logjac end + +################### +# a equivalent definition of AffineCoupling using Bijectors.Coupling +# (see https://github.com/TuringLang/Bijectors.jl/blob/74d52d4eda72a6149b1a89b72524545525419b3f/src/bijectors/coupling.jl#L188C1-L188C1) +################### + +# struct AffineCoupling <: Bijectors.Bijector +# dim::Int +# mask::Bijectors.PartitionMask +# s::Flux.Chain +# t::Flux.Chain +# end + +# # let params track field s and t +# @functor AffineCoupling (s, t) + +# function AffineCoupling(dim, mask, s, t) +# return Bijectors.Coupling(θ -> Bijectors.Shift(t(θ)) ∘ Bijectors.Scale(s(θ)), mask) +# end + +# function AffineCoupling( +# dim::Int, # dimension of input +# hdims::Int, # dimension of hidden units for s and t +# mask_idx::AbstractVector, # index of dimensione that one wants to apply transformations on +# ) +# cdims = length(mask_idx) # dimension of parts used to construct coupling law +# s = MLP_3layer(cdims, hdims, cdims) +# t = MLP_3layer(cdims, hdims, cdims) +# mask = PartitionMask(dim, mask_idx) +# return AffineCoupling(dim, mask, s, t) +# end From 2678198febaa2442e32e13383c39f716c9dd862b Mon Sep 17 00:00:00 2001 From: Zuheng Date: Mon, 31 Jul 2023 17:07:13 -0700 Subject: [PATCH 27/40] implement ham flow --- example/Hamiltonian/hamflow.jl | 0 example/HamiltonianVI/hamflow.jl | 79 ++++++++++++++++++++++++++++++++ example/Project.toml | 1 + 3 files changed, 80 insertions(+) delete mode 100644 example/Hamiltonian/hamflow.jl create mode 100644 example/HamiltonianVI/hamflow.jl diff --git a/example/Hamiltonian/hamflow.jl b/example/Hamiltonian/hamflow.jl deleted file mode 100644 index e69de29b..00000000 diff --git a/example/HamiltonianVI/hamflow.jl b/example/HamiltonianVI/hamflow.jl new file mode 100644 index 00000000..7a82f903 --- /dev/null +++ b/example/HamiltonianVI/hamflow.jl @@ -0,0 +1,79 @@ +using Functors +using Bijectors +using Bijectors: partition, combine, PartitionMask +using SimpleUnPack: @unpack + +struct LeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector + "dimention of the target space" + dim::I + "leapfrpg step size" + ϵ::AbstractVector{T} + "number of leapfrog steps" + L::I + "score of the target distribution" + ∇logp + "score of the momentum distribution" + ∇logm +end +@functor LeapFrog (ϵ,) + +function LeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇logp, ∇logm) + return LeapFrog(dim, ϵ .* one.(ϵ), L, ∇logp, ∇logm) +end + +function Bijectors.transform(lf::LeapFrog, z::AbstractVector) + @unpack d, ϵ, L, ∇logp, ∇logm = lf + @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" + mask = PartitionMask(n, 1:(lf.dim)) + x, ρ, emp = partition(mask, z) + + ρ += ϵ / 2 .* ∇logp(x) + for i in 1:(lf.L - 1) + x -= ϵ .* ∇logm(ρ) + ρ += ϵ .* ∇logp(x) + end + x -= ϵ .* ∇logm(ρ) + ρ += ϵ / 2 .* ∇logp(x) + return combine(mask, x, ρ, emp) +end + +# leapfrog composes shear transformations, hence has unit jacobian +Bijector.logabsdetjac(lf::LeapFrog, z::AbstractVector) = zero(eltype(z)) + +struct SurrogateLeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector + "dimention of the target space" + dim::I + "leapfrpg step size" + ϵ::AbstractVector{T} + "number of leapfrog steps" + L::I + "trainable surrogate of the score of the target distribution, e.g., coreset score or some neural net" + ∇S + "score of the momentum distribution" + ∇logm +end + +@functor SurrogateLeapFrog (ϵ, ∇S) + +function SurrogateLeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇S, ∇logm) + return SurrogateLeapFrog(dim, ϵ .* one.(ϵ), L, ∇S, ∇logm) +end + +function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) + @unpack d, ϵ, L, ∇S, ∇logm = slf + @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" + mask = PartitionMask(n, 1:(lf.dim)) + x, ρ, emp = partition(mask, z) + + ρ += ϵ / 2 .* ∇S(x) + for i in 1:(lf.L - 1) + x -= ϵ .* ∇logm(ρ) + ρ += ϵ .* ∇S(x) + end + x -= ϵ .* ∇logm(ρ) + ρ += ϵ / 2 .* ∇S(x) + return combine(mask, x, ρ, emp) +end + +# leapfrog composes shear transformations, hence has unit jacobian +Bijector.logabsdetjac(slf::SurrogateLeapFrog, z::AbstractVector) = zero(eltype(z)) \ No newline at end of file diff --git a/example/Project.toml b/example/Project.toml index d462c5ee..a9985022 100644 --- a/example/Project.toml +++ b/example/Project.toml @@ -14,4 +14,5 @@ Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +SimpleUnPack = "ce78b400-467f-4804-87d8-8f486da07d0a" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" From ab7ac649dc76ea6dc870e7f7f6c0591fc5ec66ca Mon Sep 17 00:00:00 2001 From: Zuheng Date: Mon, 31 Jul 2023 17:57:29 -0700 Subject: [PATCH 28/40] finish hamflow implementation --- example/HamiltonianVI/hamflow.jl | 37 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/example/HamiltonianVI/hamflow.jl b/example/HamiltonianVI/hamflow.jl index 7a82f903..2defa97f 100644 --- a/example/HamiltonianVI/hamflow.jl +++ b/example/HamiltonianVI/hamflow.jl @@ -6,7 +6,7 @@ using SimpleUnPack: @unpack struct LeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector "dimention of the target space" dim::I - "leapfrpg step size" + "leapfrog step size" ϵ::AbstractVector{T} "number of leapfrog steps" L::I @@ -21,6 +21,11 @@ function LeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇logp, ∇logm) return LeapFrog(dim, ϵ .* one.(ϵ), L, ∇logp, ∇logm) end +function Bijectors.inverse(lf::LeapFrog) + @unpack d, ϵ, L, ∇logp, ∇logm = lf + return LeapFrog(d, -ϵ, L, ∇logp, ∇logm) +end + function Bijectors.transform(lf::LeapFrog, z::AbstractVector) @unpack d, ϵ, L, ∇logp, ∇logm = lf @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" @@ -40,15 +45,28 @@ end # leapfrog composes shear transformations, hence has unit jacobian Bijector.logabsdetjac(lf::LeapFrog, z::AbstractVector) = zero(eltype(z)) -struct SurrogateLeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector +function Bijectors.with_logabsdet_jacobian(lf::LeapFrog, z::AbstractVector) + return Bijectors.transform(lf, z), zero(eltype(z)) +end + +abstract type TrainableScore end +struct CoresetScore{T<:AbstractVector} <: TrainableScore + "coreset weights" + w::T + "weighted coreset score function of the target, ∇logpw(w, x)" + ∇logpw +end + +struct SurrogateLeapFrog{T<:Real,I<:Int,H<:Union{TrainableScore,Flux.Chain}} <: + Bijectors.Bijector "dimention of the target space" dim::I - "leapfrpg step size" + "leapfrog step size" ϵ::AbstractVector{T} "number of leapfrog steps" L::I "trainable surrogate of the score of the target distribution, e.g., coreset score or some neural net" - ∇S + ∇S::H "score of the momentum distribution" ∇logm end @@ -59,6 +77,11 @@ function SurrogateLeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇S, ∇log return SurrogateLeapFrog(dim, ϵ .* one.(ϵ), L, ∇S, ∇logm) end +function Bijectors.inverse(slf::SurrogateLeapFrogeapFrog) + @unpack d, ϵ, L, ∇S, ∇logm = slf + return SurrogateLeapFrog(d, -ϵ, L, ∇S, ∇logm) +end + function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) @unpack d, ϵ, L, ∇S, ∇logm = slf @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" @@ -76,4 +99,8 @@ function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) end # leapfrog composes shear transformations, hence has unit jacobian -Bijector.logabsdetjac(slf::SurrogateLeapFrog, z::AbstractVector) = zero(eltype(z)) \ No newline at end of file +Bijector.logabsdetjac(slf::SurrogateLeapFrog, z::AbstractVector) = zero(eltype(z)) + +function Bijectors.with_logabsdet_jacobian(slf::SurrogateLeapFrog, z::AbstractVector) + return Bijectors.transform(slf, z), zero(eltype(z)) +end From 308ab61129e94e458cfe118ad4be8c65e94c8e41 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Mon, 31 Jul 2023 18:04:48 -0700 Subject: [PATCH 29/40] minor update --- example/HamiltonianVI/hamflow.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/example/HamiltonianVI/hamflow.jl b/example/HamiltonianVI/hamflow.jl index 2defa97f..9194be52 100644 --- a/example/HamiltonianVI/hamflow.jl +++ b/example/HamiltonianVI/hamflow.jl @@ -56,6 +56,7 @@ struct CoresetScore{T<:AbstractVector} <: TrainableScore "weighted coreset score function of the target, ∇logpw(w, x)" ∇logpw end +@functor CoresetScore (w,) struct SurrogateLeapFrog{T<:Real,I<:Int,H<:Union{TrainableScore,Flux.Chain}} <: Bijectors.Bijector From ceafbde22c318461155301cf3717680ec48c1015 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Thu, 3 Aug 2023 11:24:25 -0700 Subject: [PATCH 30/40] rename hamflow.jl to hamiltonian_layer.jl --- example/HamiltonianVI/hamiltonian_layer.jl | 107 +++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 example/HamiltonianVI/hamiltonian_layer.jl diff --git a/example/HamiltonianVI/hamiltonian_layer.jl b/example/HamiltonianVI/hamiltonian_layer.jl new file mode 100644 index 00000000..9194be52 --- /dev/null +++ b/example/HamiltonianVI/hamiltonian_layer.jl @@ -0,0 +1,107 @@ +using Functors +using Bijectors +using Bijectors: partition, combine, PartitionMask +using SimpleUnPack: @unpack + +struct LeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector + "dimention of the target space" + dim::I + "leapfrog step size" + ϵ::AbstractVector{T} + "number of leapfrog steps" + L::I + "score of the target distribution" + ∇logp + "score of the momentum distribution" + ∇logm +end +@functor LeapFrog (ϵ,) + +function LeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇logp, ∇logm) + return LeapFrog(dim, ϵ .* one.(ϵ), L, ∇logp, ∇logm) +end + +function Bijectors.inverse(lf::LeapFrog) + @unpack d, ϵ, L, ∇logp, ∇logm = lf + return LeapFrog(d, -ϵ, L, ∇logp, ∇logm) +end + +function Bijectors.transform(lf::LeapFrog, z::AbstractVector) + @unpack d, ϵ, L, ∇logp, ∇logm = lf + @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" + mask = PartitionMask(n, 1:(lf.dim)) + x, ρ, emp = partition(mask, z) + + ρ += ϵ / 2 .* ∇logp(x) + for i in 1:(lf.L - 1) + x -= ϵ .* ∇logm(ρ) + ρ += ϵ .* ∇logp(x) + end + x -= ϵ .* ∇logm(ρ) + ρ += ϵ / 2 .* ∇logp(x) + return combine(mask, x, ρ, emp) +end + +# leapfrog composes shear transformations, hence has unit jacobian +Bijector.logabsdetjac(lf::LeapFrog, z::AbstractVector) = zero(eltype(z)) + +function Bijectors.with_logabsdet_jacobian(lf::LeapFrog, z::AbstractVector) + return Bijectors.transform(lf, z), zero(eltype(z)) +end + +abstract type TrainableScore end +struct CoresetScore{T<:AbstractVector} <: TrainableScore + "coreset weights" + w::T + "weighted coreset score function of the target, ∇logpw(w, x)" + ∇logpw +end +@functor CoresetScore (w,) + +struct SurrogateLeapFrog{T<:Real,I<:Int,H<:Union{TrainableScore,Flux.Chain}} <: + Bijectors.Bijector + "dimention of the target space" + dim::I + "leapfrog step size" + ϵ::AbstractVector{T} + "number of leapfrog steps" + L::I + "trainable surrogate of the score of the target distribution, e.g., coreset score or some neural net" + ∇S::H + "score of the momentum distribution" + ∇logm +end + +@functor SurrogateLeapFrog (ϵ, ∇S) + +function SurrogateLeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇S, ∇logm) + return SurrogateLeapFrog(dim, ϵ .* one.(ϵ), L, ∇S, ∇logm) +end + +function Bijectors.inverse(slf::SurrogateLeapFrogeapFrog) + @unpack d, ϵ, L, ∇S, ∇logm = slf + return SurrogateLeapFrog(d, -ϵ, L, ∇S, ∇logm) +end + +function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) + @unpack d, ϵ, L, ∇S, ∇logm = slf + @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" + mask = PartitionMask(n, 1:(lf.dim)) + x, ρ, emp = partition(mask, z) + + ρ += ϵ / 2 .* ∇S(x) + for i in 1:(lf.L - 1) + x -= ϵ .* ∇logm(ρ) + ρ += ϵ .* ∇S(x) + end + x -= ϵ .* ∇logm(ρ) + ρ += ϵ / 2 .* ∇S(x) + return combine(mask, x, ρ, emp) +end + +# leapfrog composes shear transformations, hence has unit jacobian +Bijector.logabsdetjac(slf::SurrogateLeapFrog, z::AbstractVector) = zero(eltype(z)) + +function Bijectors.with_logabsdet_jacobian(slf::SurrogateLeapFrog, z::AbstractVector) + return Bijectors.transform(slf, z), zero(eltype(z)) +end From 5589f36ebb3a21a40e5c388256480beced26578f Mon Sep 17 00:00:00 2001 From: Zuheng Date: Thu, 3 Aug 2023 11:24:57 -0700 Subject: [PATCH 31/40] upadting readme --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index f9fbf663..9ab359dd 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,36 @@ [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://turinglang.github.io/NormalizingFlows.jl/dev/) [![Build Status](https://github.com/TuringLang/NormalizingFlows.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/TuringLang/NormalizingFlows.jl/actions/workflows/CI.yml?query=branch%3Amain) + + +A normalizing flow library for Julia. + +The purpose of this package is to provide a simple and flexible interface for +variational inference (VI) and normalizing flows (NF) for Bayesian computation or generative modeling. +The key focus is to ensure modularity and extensibility, so that users can easily +construct (e.g., define customized flow layers) and combine various components +(e.g., choose different VI objectives or gradient estimates) +for variational approximation of general target distributions, +without being tied to specific probabilistic programming frameworks or applications. + +See the [documentation](https://turinglang.org/NormalizingFlows.jl/dev/) for more. + +## Installation +To install the package, run the following command in the Julia REPL: +```julia +# install the package +] # enter Pkg mode +(@v1.9) pkg> add git@github.com:TuringLang/NormalizingFlows.jl.git +``` +Then simply run the following command to use the package: +```julia +using NormalizingFlows +``` + + +## Quick recap of normalizing flows + + +## Current status and TODOs + + From de01e0e478a7a3d0560c5fbf3f385da9149be7bc Mon Sep 17 00:00:00 2001 From: Zuheng Date: Thu, 3 Aug 2023 11:25:46 -0700 Subject: [PATCH 32/40] rm hamflow.jl --- example/HamiltonianVI/hamflow.jl | 107 ------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 example/HamiltonianVI/hamflow.jl diff --git a/example/HamiltonianVI/hamflow.jl b/example/HamiltonianVI/hamflow.jl deleted file mode 100644 index 9194be52..00000000 --- a/example/HamiltonianVI/hamflow.jl +++ /dev/null @@ -1,107 +0,0 @@ -using Functors -using Bijectors -using Bijectors: partition, combine, PartitionMask -using SimpleUnPack: @unpack - -struct LeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector - "dimention of the target space" - dim::I - "leapfrog step size" - ϵ::AbstractVector{T} - "number of leapfrog steps" - L::I - "score of the target distribution" - ∇logp - "score of the momentum distribution" - ∇logm -end -@functor LeapFrog (ϵ,) - -function LeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇logp, ∇logm) - return LeapFrog(dim, ϵ .* one.(ϵ), L, ∇logp, ∇logm) -end - -function Bijectors.inverse(lf::LeapFrog) - @unpack d, ϵ, L, ∇logp, ∇logm = lf - return LeapFrog(d, -ϵ, L, ∇logp, ∇logm) -end - -function Bijectors.transform(lf::LeapFrog, z::AbstractVector) - @unpack d, ϵ, L, ∇logp, ∇logm = lf - @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" - mask = PartitionMask(n, 1:(lf.dim)) - x, ρ, emp = partition(mask, z) - - ρ += ϵ / 2 .* ∇logp(x) - for i in 1:(lf.L - 1) - x -= ϵ .* ∇logm(ρ) - ρ += ϵ .* ∇logp(x) - end - x -= ϵ .* ∇logm(ρ) - ρ += ϵ / 2 .* ∇logp(x) - return combine(mask, x, ρ, emp) -end - -# leapfrog composes shear transformations, hence has unit jacobian -Bijector.logabsdetjac(lf::LeapFrog, z::AbstractVector) = zero(eltype(z)) - -function Bijectors.with_logabsdet_jacobian(lf::LeapFrog, z::AbstractVector) - return Bijectors.transform(lf, z), zero(eltype(z)) -end - -abstract type TrainableScore end -struct CoresetScore{T<:AbstractVector} <: TrainableScore - "coreset weights" - w::T - "weighted coreset score function of the target, ∇logpw(w, x)" - ∇logpw -end -@functor CoresetScore (w,) - -struct SurrogateLeapFrog{T<:Real,I<:Int,H<:Union{TrainableScore,Flux.Chain}} <: - Bijectors.Bijector - "dimention of the target space" - dim::I - "leapfrog step size" - ϵ::AbstractVector{T} - "number of leapfrog steps" - L::I - "trainable surrogate of the score of the target distribution, e.g., coreset score or some neural net" - ∇S::H - "score of the momentum distribution" - ∇logm -end - -@functor SurrogateLeapFrog (ϵ, ∇S) - -function SurrogateLeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇S, ∇logm) - return SurrogateLeapFrog(dim, ϵ .* one.(ϵ), L, ∇S, ∇logm) -end - -function Bijectors.inverse(slf::SurrogateLeapFrogeapFrog) - @unpack d, ϵ, L, ∇S, ∇logm = slf - return SurrogateLeapFrog(d, -ϵ, L, ∇S, ∇logm) -end - -function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) - @unpack d, ϵ, L, ∇S, ∇logm = slf - @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" - mask = PartitionMask(n, 1:(lf.dim)) - x, ρ, emp = partition(mask, z) - - ρ += ϵ / 2 .* ∇S(x) - for i in 1:(lf.L - 1) - x -= ϵ .* ∇logm(ρ) - ρ += ϵ .* ∇S(x) - end - x -= ϵ .* ∇logm(ρ) - ρ += ϵ / 2 .* ∇S(x) - return combine(mask, x, ρ, emp) -end - -# leapfrog composes shear transformations, hence has unit jacobian -Bijector.logabsdetjac(slf::SurrogateLeapFrog, z::AbstractVector) = zero(eltype(z)) - -function Bijectors.with_logabsdet_jacobian(slf::SurrogateLeapFrog, z::AbstractVector) - return Bijectors.transform(slf, z), zero(eltype(z)) -end From c858b109381dfbc0adee12c29fff02f6eb076b0c Mon Sep 17 00:00:00 2001 From: Zuheng Date: Tue, 15 Aug 2023 22:36:24 -0700 Subject: [PATCH 33/40] fix minor bugs in affine coupling layer --- example/RealNVP/AffineCoupling.jl | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/example/RealNVP/AffineCoupling.jl b/example/RealNVP/AffineCoupling.jl index 50e75220..1efd01ed 100644 --- a/example/RealNVP/AffineCoupling.jl +++ b/example/RealNVP/AffineCoupling.jl @@ -6,7 +6,7 @@ using Bijectors: partition, combine, PartitionMask include("../util.jl") """ -Affinecoupling layer for RealNVP "(http://proceedings.mlr.press/v118/fjelde20a/fjelde20a.pdf)" +Affinecoupling layer """ struct AffineCoupling <: Bijectors.Bijector dim::Int @@ -30,14 +30,6 @@ function AffineCoupling( return AffineCoupling(dim, mask, s, t) end -## scaling parameterize using exp -# function Bijectors.transform(af::AffineCoupling, x::AbstractVector) -# # partition vector using 'af.mask::PartitionMask` -# x₁, x₂, x₃ = Bijectors.partition(af.Mask, x) -# y₁ = x₁ .* exp.(af.s(x₂)) .+ af.t(x₂) -# return Bijectors.combine(af.mask, y₁, x₂, x₃) -# end - function Bijectors.transform(af::AffineCoupling, x::AbstractVector) # partition vector using 'af.mask::PartitionMask` x₁, x₂, x₃ = partition(af.mask, x) @@ -51,7 +43,7 @@ end function Bijectors.with_logabsdet_jacobian(af::AffineCoupling, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(af.mask, x) - y_1 = exp.(af.s(x_2)) .* x_1 .+ af.t(x_2) + y_1 = af.s(x_2) .* x_1 .+ af.t(x_2) logjac = sum(log ∘ abs, af.s(x_2)) return combine(af.mask, y_1, x_2, x_3), logjac end @@ -75,7 +67,7 @@ function Bijectors.logabsdetjac(af::AffineCoupling, x::AbstractVector) end ################### -# a equivalent definition of AffineCoupling using Bijectors.Coupling +# an equivalent definition of AffineCoupling using Bijectors.Coupling # (see https://github.com/TuringLang/Bijectors.jl/blob/74d52d4eda72a6149b1a89b72524545525419b3f/src/bijectors/coupling.jl#L188C1-L188C1) ################### From baef6a964e39175e6182d055bbce0668a36239ce Mon Sep 17 00:00:00 2001 From: Zuheng Date: Tue, 15 Aug 2023 22:36:50 -0700 Subject: [PATCH 34/40] test affine coupling flow on banana --- example/RealNVP/main.jl | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 example/RealNVP/main.jl diff --git a/example/RealNVP/main.jl b/example/RealNVP/main.jl new file mode 100644 index 00000000..3e94bf06 --- /dev/null +++ b/example/RealNVP/main.jl @@ -0,0 +1,56 @@ +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using FunctionChains +using NormalizingFlows +using Zygote +using Flux: f32 +using Plots +include("../common.jl") +include("AffineCoupling.jl") + +Random.seed!(123) +rng = Random.default_rng() + +###################################### +# 2d Banana as the target distribution +###################################### +include("../targets/banana.jl") + +# create target p +p = Banana(2, 1.0f-1, 100.0f0) +logp = Base.Fix1(logpdf, p) + +###################################### +# learn the target using Affine coupling flow +###################################### +d = 2 +hdims = 20 +ls = [AffineCoupling(d, hdims, [1]), AffineCoupling(d, hdims, [2])] +Ls = reduce(vcat, fill(ls, 2)) +q0 = MvNormal(zeros(Float32, 2), I) +flow = create_flow(Ls, q0) +flow_untrained = deepcopy(flow) + +# train the flow +sample_per_iter = 10 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(), + callback=cb, + ADbackend=AutoZygote(), + hasconverged=checkconv, +) +losses = map(x -> x.loss, stats) + +###################################### +# evaluate trained flow (one can see that the performance is better than the plannar flow) +###################################### +plot(losses; label="Loss", linewidth=2) # plot the loss +compare_trained_and_untrained_flow(flow_trained, flow_untrained, p, 1000) From 03236802cdd5b90689a390830b96d670beab8f1a Mon Sep 17 00:00:00 2001 From: Zuheng Date: Tue, 15 Aug 2023 22:37:09 -0700 Subject: [PATCH 35/40] rename simple flow run files --- .../planar_radial_flow/planar_flow_main.jl | 58 +++++++++++++++++++ .../planar_radial_flow/radial_flow_main.jl | 55 ++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 example/planar_radial_flow/planar_flow_main.jl create mode 100644 example/planar_radial_flow/radial_flow_main.jl diff --git a/example/planar_radial_flow/planar_flow_main.jl b/example/planar_radial_flow/planar_flow_main.jl new file mode 100644 index 00000000..c79077d9 --- /dev/null +++ b/example/planar_radial_flow/planar_flow_main.jl @@ -0,0 +1,58 @@ +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using FunctionChains +using NormalizingFlows +using Zygote +using Flux: f32 +using Plots +include("../common.jl") + +Random.seed!(123) +rng = Random.default_rng() + +###################################### +# 2d Banana as the target distribution +###################################### +include("../targets/banana.jl") + +# create target p +p = Banana(2, 1.0f-1, 100.0f0) +logp = Base.Fix1(logpdf, p) + +###################################### +# learn the target using planar flow +###################################### +function create_planar_flow(n_layers::Int, q₀) + d = length(q₀) + Ls = [f32(PlanarLayer(d)) for _ in 1:n_layers] + ts = fchain(Ls) + return transformed(q₀, ts) +end + +# create a 10-layer planar flow +flow = create_planar_flow(20, MvNormal(zeros(Float32, 2), I)) +flow_untrained = deepcopy(flow) + +# train the flow +sample_per_iter = 10 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(), + callback=cb, + ADbackend=AutoZygote(), + hasconverged=checkconv, +) +losses = map(x -> x.loss, stats) + +###################################### +# evaluate trained flow +###################################### +plot(losses; label="Loss", linewidth=2) # plot the loss +compare_trained_and_untrained_flow(flow_trained, flow_untrained, p, 1000) diff --git a/example/planar_radial_flow/radial_flow_main.jl b/example/planar_radial_flow/radial_flow_main.jl new file mode 100644 index 00000000..70085d11 --- /dev/null +++ b/example/planar_radial_flow/radial_flow_main.jl @@ -0,0 +1,55 @@ +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using FunctionChains +using NormalizingFlows +using Zygote +using Flux: f32 +using Plots +include("../common.jl") + +Random.seed!(123) +rng = Random.default_rng() + +###################################### +# 2d Banana as the target distribution +###################################### +include("../targets/banana.jl") + +# create target p +p = Banana(2, 1.0f-1, 100.0f0) +logp = Base.Fix1(logpdf, p) + +###################################### +# learn the target using radial flow +###################################### +function create_radial_flow(n_layers::Int, q₀) + d = length(q₀) + Ls = [f32(RadialLayer(d)) for _ in 1:n_layers] + ts = fchain(Ls) + return transformed(q₀, ts) +end + +# create a 20-layer radial flow +flow = create_radial_flow(10, MvNormal(zeros(Float32, 2), I)) +flow_untrained = deepcopy(flow) + +# train the flow +sample_per_iter = 10 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(), + callback=cb, +) +losses = map(x -> x.loss, stats) + +###################################### +# evaluate trained flow +###################################### +plot(losses; label="Loss", linewidth=2) # plot the loss +compare_trained_and_untrained_flow(flow_trained, flow_untrained, p, 1000; legend=:bottom) From 144b06a4ef677d914b6a66e1c151170ed410f3d6 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Tue, 15 Aug 2023 23:02:44 -0700 Subject: [PATCH 36/40] update loglikelihood to fit in optimize interface --- src/objectives/loglikelihood.jl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/objectives/loglikelihood.jl b/src/objectives/loglikelihood.jl index 8564793d..1b1b29e5 100644 --- a/src/objectives/loglikelihood.jl +++ b/src/objectives/loglikelihood.jl @@ -21,4 +21,30 @@ function loglikelihood( ) llhs = map(x -> logpdf(flow, x), eachcol(xs)) return mean(llhs) +end + +function loglikelihood( + rng::AbstractRNG, + flow::Bijectors.UnivariateTransformed, + fulldata::AbstractVector, + batchsize::Int, +) + N = length(fulldata) + @assert batchsize <= N + idx = sample(rng, 1:N, batchsize; replace=false) + xs = @view(fulldata[idx]) + return loglikelihood(flow, xs) +end + +function loglikelihood( + rng::AbstractRNG, + flow::Bijectors.MultivariateTransformed, + fulldata::AbstractMatrix, + batchsize::Int, +) + N = size(fulldata, 2) + @assert batchsize <= N + idx = sample(rng, 1:N, batchsize; replace=false) + xs = @view(fulldata[:, idx]) + return loglikelihood(flow, xs) end \ No newline at end of file From f70a4b75a353d8057ea1032a27ea780d9b1cde51 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Thu, 17 Aug 2023 01:59:04 -0700 Subject: [PATCH 37/40] fix minor bugs in nsf_layer --- example/HamiltonianVI/data/bank_dat.csv | 401 ++++++++++++++++++ example/HamiltonianVI/hamiltonian_layer.jl | 67 +-- example/HamiltonianVI/main.jl | 161 +++++++ example/Project.toml | 5 + example/RealNVP/main.jl | 3 +- .../data/DatasaurusDozen-Wide.tsv | 144 +++++++ example/neural_spline_flow/data/data_proc.jl | 7 + .../main.jl} | 40 +- example/neural_spline_flow/nsf_layer.jl | 102 +++-- example/planar_radial_flow/radial_flow.jl | 55 --- 10 files changed, 839 insertions(+), 146 deletions(-) create mode 100644 example/HamiltonianVI/data/bank_dat.csv create mode 100644 example/HamiltonianVI/main.jl create mode 100644 example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv create mode 100644 example/neural_spline_flow/data/data_proc.jl rename example/{planar_radial_flow/planar_flow.jl => neural_spline_flow/main.jl} (55%) delete mode 100644 example/planar_radial_flow/radial_flow.jl diff --git a/example/HamiltonianVI/data/bank_dat.csv b/example/HamiltonianVI/data/bank_dat.csv new file mode 100644 index 00000000..a8159d20 --- /dev/null +++ b/example/HamiltonianVI/data/bank_dat.csv @@ -0,0 +1,401 @@ +"","age","marital","balance","housing","duration","campaign","pdays","previous","y" +"1",0.19806976350248,-1.04629924020543,4457.41775891603,-0.959567210136544,121.830353246711,-0.723428206559773,0.353721396016046,0.452003571795249,"1" +"2",104.146940630705,-0.702269515361112,-0.260956339631889,-1.04629924020543,3678.39336905394,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"3",-0.689720178893626,-0.476696519712976,4730.53644018208,-0.702269515361112,33.7987805129382,-1.03860930317821,-2.99624765331573,-0.959567210136544,"1" +"4",118.986334056931,-0.00208617190785308,0.804669047638875,-0.484837871827967,779.493453783607,-0.702269515361112,20.6706339632953,-1.04373592786303,"1" +"5",-0.954019995878704,0.998749217771909,508.810513174902,-0.280242426288262,-0.287920489113158,-0.468555167597986,242.406856709252,-0.212029015981627,"1" +"6",39.0072109343771,-4.02497038533371,21.0058768415407,-0.998749217771909,856.653626849399,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"7",-0.102194322423417,0.452003571795249,846.923553911768,-4.02497038533371,-0.454174089108914,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"8",10.5678514945477,-0.723428206559773,20.9269181905968,-2.2068409681768,317.954055081304,-3.65774950282621,-1.04886255254784,-0.998749217771909,"1" +"9",58.9262038485426,1.03953114431459,134.347384693829,-0.723428206559773,5.23853266501029,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"10",-1.45442420778125,-1.04373592786303,13116.5734769985,1.03953114431459,220.019511042995,-0.722978766190891,1.94128505843918,0.452003571795249,"1" +"11",159.982675970118,-0.702269515361112,111.28238672202,-1.04629924020543,1003.74296386077,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"12",-0.703203389960085,-0.476696519712976,2135.50416916935,-0.702269515361112,93.4721739204061,-1.04117261552062,720.098186013546,1.03953114431459,"1" +"13",98.9953505124201,-0.280242426288262,-0.545899260851401,-0.484837871827967,1901.52584965181,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"14",-0.925823560112203,0.998749217771909,7347.72598375223,-0.280242426288262,-0.537359893842643,-0.468555167597986,907.117991702265,-0.702269515361112,"1" +"15",15.9659074635414,-4.02497038533371,-0.564396519832509,0.998749217771909,3727.35886384123,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"16",-0.175466491458331,-2.2068409681768,582.193684246846,-4.02497038533371,-0.838670940470291,2.99624765331573,780.687889380256,0.554226336852964,"1" +"17",7.78628895074358,-0.723428206559773,-4.49852446451824,0.452003571795249,261.576397652663,-3.93316516470684,-0.190152917840769,6.99124452440336,"1" +"18",116.853658479313,1.03953114431459,-0.280242426288262,-0.723877646928655,3.5369900729773,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"19",-0.53637200151251,-1.04373592786303,104.86866786605,-0.959567210136544,71.2059149494767,-0.723428206559773,2.72285486147826,0.452003571795249,"1" +"20",82.8761843109289,-0.702269515361112,27.8314411721915,-1.04373592786303,1848.6848020958,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"21",-0.701855068853439,-0.476696519712976,4863.47866718068,-0.702269515361112,19.8443869776534,-1.04373592786303,356.553470744572,1.03953114431459,"1" +"22",69.0088751956531,-0.00208617190785308,-0.334212847108,-0.476696519712976,609.327403225395,0.278211483397858,-4.20858082658746,-1.04629924020543,"1" +"23",-0.96683655759075,0.998749217771909,2809.77271914816,-0.00208617190785308,-0.233538204478442,-0.468555167597986,-4.86568550814886,-1.1925100147406,"1" +"24",14.9854264647824,-4.11677560596058,8.04320632597751,0.998749217771909,1552.33985419839,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"25",-0.183607843573321,0.452003571795249,91.462944367982,-4.11677560596058,-0.397781217575912,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"26",12.5149452752105,-0.723877646928655,-0.484837871827967,-2.2068409681768,500.323520850472,-3.84135994407996,-1.04886255254784,-0.998749217771909,"1" +"27",64.9186991551741,1.03953114431459,237.543355068961,-0.723877646928655,0.410710860820979,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"28",0.657095866636848,-1.04373592786303,186.766103723347,-0.959567210136544,67.5898836425314,-0.723428206559773,-0.492979223942957,-2.2068409681768,"1" +"29",112.123474250621,-0.702269515361112,8.00151351678674,-1.04629924020543,1145.56535278438,1.03953114431459,23.084882941666,-0.722529325822009,"1" +"30",-0.703652830328967,-0.484837871827967,-552.587660742392,-1.1925100147406,76.4882081044345,-1.04117261552062,-2.99624765331573,-0.959567210136544,"1" +"31",90.9989570946156,-0.00208617190785308,-0.236234846691733,-0.476696519712976,1388.36885343721,-0.212029015981627,28.2904672753258,-0.897627124345702,"1" +"32",-0.923260247769794,0.998749217771909,3189.60140649387,-0.00208617190785308,-0.382302966578369,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"33",11.553742969126,-4.02497038533371,3.04218394593721,-0.998749217771909,308.900677729789,-0.00208617190785308,-0.67668640819605,-0.476696519712976,"1" +"34",-0.142901082998369,0.452003571795249,0.768451982777342,-4.02497038533371,0.991534072009863,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"35",13.9057265471126,-0.723428206559773,-0.484837871827967,0.452003571795249,462.575002398252,-3.84135994407996,-1.04886255254784,-0.998749217771909,"1" +"36",68.9136960262617,1.03953114431459,327.109668979452,-0.723877646928655,1.4935106921147,8.42853719171141,-1.68275051412008,-4.11677560596058,"1" +"37",-0.628177222139383,-1.04373592786303,8799.97935778829,1.03953114431459,191.369416841813,-0.722978766190891,-0.492979223942957,-2.2068409681768,"1" +"38",128.076541490454,-0.702269515361112,3.68666814732368,-1.04373592786303,1177.52532775308,9.03592456211912,99.5778528962784,-0.723428206559773,"1" +"39",-0.709944995493314,-0.484837871827967,574.762424205759,-0.702269515361112,158.929296227367,-1.04117261552062,-2.99624765331573,-0.959567210136544,"1" +"40",110.989940639127,-0.00208617190785308,0.848264763420425,-0.484837871827967,699.728117584445,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"41",-0.946330058851477,0.998749217771909,3669.38501156214,-0.280242426288262,-0.318482434197131,-0.460413815482995,-4.86568550814886,-1.1925100147406,"1" +"42",14.4951859654029,-4.02497038533371,0.194343933520608,0.998749217771909,164.965596209307,1.38869509999419,-0.639832297947729,-0.460413815482995,"1" +"43",-0.020780801273513,0.452003571795249,-46.2946359576532,-4.02497038533371,3.09857681747021,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"44",18.0780703628187,-0.723428206559773,4.29413581967141,-2.2068409681768,222.847398201684,-3.93316516470684,-0.0414808019810366,2.99624765331573,"1" +"45",64.9186991551741,1.03953114431459,19.4688516347207,-0.723877646928655,2.24251508669382,5.76969265173935,48.8120209219668,-3.93316516470684,"1" +"46",-0.628177222139383,-1.04629924020543,138.826141270295,-0.959567210136544,72.8748524757592,-0.723428206559773,-0.492979223942957,-2.2068409681768,"1" +"47",77.5584952309848,-0.702269515361112,240.635942585285,-1.04629924020543,947.813007665542,3.03862949876572,25.5882892310897,-0.722079885453127,"1" +"48",-0.705450591804495,-0.476696519712976,1539.92299221561,-0.702269515361112,76.120987221927,-1.04373592786303,724.093182884634,1.03953114431459,"1" +"49",120.985432411383,-0.00208617190785308,4.15749419949826,-0.484837871827967,596.033180525535,-0.702269515361112,4.23749947108492,-1.03348267849339,"1" +"50",-0.943766746509068,0.998749217771909,2613.86108041195,-0.280242426288262,0.0743284482056978,-0.468555167597986,-4.86568550814886,-1.1925100147406,"1" +"51",15.4756669641619,-4.02497038533371,17.612051300191,0.998749217771909,1762.24518141576,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"52",-0.020780801273513,0.452003571795249,1139.10689154194,-4.11677560596058,-0.0773671747747651,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"53",12.5149452752105,-0.723428206559773,2.35649401630369,-2.2068409681768,227.259562696099,-3.74955472345309,-0.820727754073427,4.99374608885955,"1" +"54",110.861163172682,-0.959567210136544,192.760198113715,-0.723877646928655,0.687516832730653,0.452003571795249,48.8120209219668,-4.02497038533371,"1" +"55",1.20792719039809,-1.04373592786303,1127.58786686449,-0.959567210136544,42.2776644939142,-0.723428206559773,0.272307874866142,0.452003571795249,"1" +"56",152.006142350202,-1.1925100147406,519.264787187846,-1.04629924020543,706.11569696474,1.03953114431459,23.9193517048073,-0.723428206559773,"1" +"57",-0.69511346332021,-0.476696519712976,1008.1540842212,-1.1925100147406,39.8579250743119,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"58",67.009776841202,-0.280242426288262,0.572757817295786,-0.484837871827967,832.670344583048,-0.212029015981627,10.2048388118317,-1.0360459908358,"1" +"59",-0.889937187318475,0.998749217771909,5156.71418727379,-0.280242426288262,-0.602528747330527,-0.460413815482995,534.879756106178,2.2391734809158,"1" +"60",16.9463884623003,-4.02497038533371,7.03838788775312,0.998749217771909,806.676167988121,-0.00208617190785308,-0.586798334419658,-0.460413815482995,"1" +"61",-0.020780801273513,0.452003571795249,1393.54171071989,-4.02497038533371,1.88356676716826,8.98874295994718,-2.95866556458768,-0.280242426288262,"1" +"62",11.6804765120693,-0.723877646928655,4.23714635486647,0.452003571795249,741.031606045799,-4.02497038533371,-0.318318534961228,2.99624765331573,"1" +"63",70.9111944618055,1.03953114431459,418.623076670607,-0.723877646928655,2.16924291765891,5.76969265173935,-1.68275051412008,-4.11677560596058,"1" +"64",0.657095866636848,-1.04373592786303,2449.93183119449,-0.959567210136544,312.08923124291,-0.722079885453127,-0.492979223942957,-2.2068409681768,"1" +"65",165.300365050063,-0.702269515361112,261.751143329466,-1.04629924020543,606.240775187549,1.03953114431459,32.8203518449803,-0.722978766190891,"1" +"66",-0.697810105533501,-0.484837871827967,5174.56347835741,-1.1925100147406,16.8148146969666,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"67",92.9980554490667,-0.00208617190785308,-0.723877646928655,-0.484837871827967,1569.17028215531,-0.702269515361112,-3.19872339969185,-1.04373592786303,"1" +"68",-0.915570310742567,0.998749217771909,4185.15238701054,-0.280242426288262,-0.518033957980719,-0.435989759138024,-4.86568550814886,-1.1925100147406,"1" +"69",34.5950464399618,-4.02497038533371,0.994097384352272,-0.998749217771909,852.655430140497,-0.00208617190785308,-0.642079499792139,-0.468555167597986,"1" +"70",-0.102194322423417,0.452003571795249,1365.10776175588,-4.11677560596058,0.181527371808562,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"71",12.2367890208301,-0.723428206559773,43.0388305349108,0.452003571795249,90.482463369223,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"72",110.861163172682,1.03953114431459,-335.736685209061,-0.723428206559773,2.62515863609837,3.1108481117673,-1.68275051412008,-4.11677560596058,"1" +"73",-0.628177222139383,-1.04373592786303,13438.170725121,-0.959567210136544,69.8151336775747,-0.722079885453127,-0.492979223942957,-2.2068409681768,"1" +"74",101.488096090733,-0.702269515361112,6.71624042801051,-1.04629924020543,1808.73483338493,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"75",-0.709495555124432,-0.476696519712976,1997.24425309081,-0.702269515361112,43.5301338993869,-1.04117261552062,330.585991082502,9.03592456211912,"1" +"76",108.990842284676,-0.280242426288262,-0.723877646928655,-0.484837871827967,694.410428504501,1.74893298153631,13.601631975026,-1.0360459908358,"1" +"77",-0.969399869933159,0.998749217771909,4329.08746853102,-0.280242426288262,-0.581854490361957,-0.468555167597986,-4.86568550814886,-1.1925100147406,"1" +"78",23.3195149542336,-4.02497038533371,-0.456737401451324,-0.998749217771909,2176.05854078715,0.832382591233372,-0.724327087297537,-0.484837871827967,"1" +"79",-0.102194322423417,0.452003571795249,151.762525791659,-4.11677560596058,1.21966887048428,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"80",14.4620390558734,-0.723428206559773,14.1370305266948,-2.2068409681768,232.161967689894,-4.02497038533371,-0.551579958120463,0.998749217771909,"1" +"81",120.848655350401,1.03953114431459,23.6411954504269,-0.723877646928655,4.71748612965091,0.452003571795249,44.890096926931,-4.02497038533371,"1" +"82",-0.169151119005015,-1.04373592786303,1395.25265722736,1.03953114431459,349.640325584266,-0.722978766190891,0.508407086200864,3.1108481117673,"1" +"83",98.8292515507612,-0.702269515361112,3.9620838092043,-1.04629924020543,562.295809605585,3.03862949876572,51.4568208884677,-0.722079885453127,"1" +"84",-0.70904611475555,-0.476696519712976,3395.79648111611,-1.1925100147406,55.4648125808804,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"85",108.990842284676,-0.00208617190785308,0.774556542923784,-0.484837871827967,1678.18290829416,0.768451982777342,-4.20858082658746,-1.04629924020543,"1" +"86",-0.954019995878704,0.998749217771909,988.594118243174,-0.280242426288262,-0.639832297947729,-0.476696519712976,247.724545789196,3.21965447967477,"1" +"87",23.3195149542336,-4.02497038533371,9.48378786241147,0.998749217771909,1050.56616723116,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"88",-0.0126394491585226,0.452003571795249,1055.27576614805,-4.11677560596058,-0.410597779287958,2.99624765331573,362.87633329997,1.6668513543746,"1" +"89",11.6804765120693,-0.723428206559773,34.7916408424255,0.452003571795249,729.265834060692,-3.84135994407996,-1.04886255254784,-0.998749217771909,"1" +"90",128.838649092576,1.03953114431459,888.428990319117,-0.723877646928655,6.87494444012337,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"91",-1.08720332527375,-1.04373592786303,3930.07817193246,1.03953114431459,505.129671782914,-0.722529325822009,-0.492979223942957,-2.2068409681768,"1" +"92",144.029608730286,-0.702269515361112,758.233776479598,-1.04629924020543,384.518448842185,1.03953114431459,25.3101329767093,-0.723428206559773,"1" +"93",-0.688821298155862,-0.484837871827967,606.668558685423,-1.1925100147406,4.78833079484616,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"94",142.975514310345,-0.00208617190785308,-0.722978766190891,-0.484837871827967,388.643306407715,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"95",-0.887373874976066,-0.998749217771909,11.0350229165703,-0.00208617190785308,-0.677585288933813,-0.476696519712976,481.702865306737,0.278211483397858,"1" +"96",18.9073504598183,-4.02497038533371,-0.415724403972777,-0.998749217771909,1364.42460887999,0.554226336852964,-0.724327087297537,-0.484837871827967,"1" +"97",0.0606327198763911,-2.2068409681768,992.524982227475,-4.11677560596058,-0.633605953077557,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"98",8.3426014595044,-0.723877646928655,-2.73999240768031,0.452003571795249,492.96991335978,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"99",90.8861788172437,1.03953114431459,919.860647064103,-0.723877646928655,0.907333339835394,0.452003571795249,43.4193754287925,-3.93316516470684,"1" +"100",0.565290646009974,-1.04373592786303,5262.40962844019,-0.959567210136544,112.929353106538,-0.722529325822009,0.117622184681324,11.0873817316835,"1" +"101",114.782318790594,-0.702269515361112,93.0131478172718,-1.04629924020543,374.530956664466,5.03772785321685,-0.55839868066867,-0.723877646928655,"1" +"102",-0.704102270697849,-0.476696519712976,4063.16646064909,-0.702269515361112,27.7396359515646,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"103",61.0124817778486,-0.00208617190785308,-0.673989765982758,-0.484837871827967,1337.85080717774,-0.212029015981627,4.42110991233867,-1.04117261552062,"1" +"104",-0.951456683536295,0.998749217771909,8173.35360414055,-0.00208617190785308,-0.59398938032177,-0.468555167597986,460.43210898696,0.768451982777342,"1" +"105",27.731679448649,-4.02497038533371,16.609796174309,-0.998749217771909,2048.11624610227,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"106",-0.118477026653398,0.452003571795249,2503.93644181443,-4.11677560596058,-0.172209731443905,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"107",16.4091328365362,-0.723428206559773,13.6811148082554,-2.2068409681768,143.918677801587,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"108",62.9212007196303,1.03953114431459,217.516104753572,-0.723428206559773,7.02963013030818,5.76969265173935,-1.68275051412008,-4.11677560596058,"1" +"109",1.29973241102496,-1.04373592786303,989.760474811962,-0.959567210136544,204.999073306453,-0.722529325822009,1.00502956521528,0.452003571795249,"1" +"110",93.5115624708171,-0.702269515361112,106.692125690676,-1.04373592786303,3578.51844727675,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"111",-0.70679891291114,-0.476696519712976,2481.15395936572,-0.702269515361112,61.5239571422541,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"112",96.996252157969,-0.280242426288262,-0.298257617597443,-0.484837871827967,2146.13954732924,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"113",-0.946330058851477,0.998749217771909,-0.959567210136544,-0.280242426288262,-0.559382471917859,-0.476696519712976,1157.04937845964,3.70989497905425,"1" +"114",14.4951859654029,-4.11677560596058,-1.16421160795626,-0.998749217771909,2743.80247345127,0.832382591233372,-0.724327087297537,-0.484837871827967,"1" +"115",-0.0614875618484651,0.452003571795249,-255.627329192693,-4.02497038533371,0.0866848151394224,14.9812382665786,-2.95866556458768,-0.280242426288262,"1" +"116",18.6343828715795,-0.723428206559773,3.18691193203271,-2.2068409681768,181.667196253807,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"117",74.9061913328932,-0.959567210136544,16.6872890909167,-0.723877646928655,4.69306207330594,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"118",-0.903592884020004,-1.04373592786303,1978.52220040615,1.03953114431459,413.89442034614,-0.722529325822009,-0.492979223942957,-2.2068409681768,"1" +"119",85.5350288509009,-0.702269515361112,29.7593508053558,-1.04629924020543,889.885553034771,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"120",-0.703203389960085,-0.476696519712976,1699.45366461394,-0.702269515361112,5.79818822174177,-1.04117261552062,256.678548967381,3.03862949876572,"1" +"121",63.0115801322997,-0.00208617190785308,-0.51129235244749,-0.484837871827967,774.175764703663,0.278211483397858,-4.20858082658746,-1.04629924020543,"1" +"122",-0.951456683536295,0.998749217771909,1096.54542938354,-0.280242426288262,-0.616461398765867,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"123",18.9073504598183,-4.02497038533371,2.89607514241988,-0.998749217771909,1418.40026445017,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"124",-0.16732513934334,0.452003571795249,56.1656284126591,-4.02497038533371,1.59903909716084,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"125",6.11735142446113,-0.723428206559773,0.0524913677614007,0.452003571795249,680.732024622123,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"126",96.8786741238752,1.03953114431459,1003.30752337823,-0.723428206559773,1.56678286114962,0.452003571795249,81.1678938810128,-4.02497038533371,"1" +"127",2.67681072042807,-1.04373592786303,1337.32520259659,-0.959567210136544,73.7093212389004,-0.721630445084245,-0.492979223942957,-2.2068409681768,"1" +"128",149.34729781023,-0.702269515361112,967.457874288243,-1.04629924020543,588.263289267654,5.03772785321685,-0.55839868066867,-0.723877646928655,"1" +"129",-0.703203389960085,-0.476696519712976,891.164924462433,-0.702269515361112,19.7525817570266,-1.03860930317821,911.858035825753,1.03953114431459,"1" +"130",112.989038993578,-0.00208617190785308,-0.172414314310496,-0.484837871827967,614.645092305339,-0.702269515361112,-4.20858082658746,-1.04629924020543,"1" +"131",-0.920696935427385,0.998749217771909,20489.798565914,-0.00208617190785308,-0.680281931147105,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"132",17.4366289616798,-4.02497038533371,16.9097037183709,0.998749217771909,738.706823936783,-0.00208617190785308,-0.538708214949289,-0.476696519712976,"1" +"133",-0.0696289139634555,0.452003571795249,-954.220040808459,-4.02497038533371,0.550644349115484,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"134",18.0780703628187,-0.723428206559773,17.0109278232864,0.452003571795249,64.9899574014898,-3.93316516470684,-0.815601129388609,20.9737335732101,"1" +"135",60.9237022840865,1.03953114431459,723.760487725915,-0.723428206559773,1.79474072036935,8.42853719171141,-1.68275051412008,-4.11677560596058,"1" +"136",1.94236895541308,-1.04629924020543,104.86866786605,-0.959567210136544,156.043572535501,-0.722079885453127,-0.492979223942957,-2.2068409681768,"1" +"137",152.006142350202,-0.702269515361112,186.287251974175,-1.04629924020543,516.353345588077,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"138",-0.701405628484557,-0.476696519712976,830.011500043076,-1.1925100147406,83.8326257545844,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"139",61.0124817778486,-0.00208617190785308,1.1404010031937,-0.476696519712976,837.988033662992,0.278211483397858,4.97194123609991,-1.02322942912375,"1" +"140",-0.920696935427385,0.998749217771909,2184.05493420495,-0.00208617190785308,-0.581854490361957,-0.476696519712976,651.868915864948,0.768451982777342,"1" +"141",21.8487934560952,-4.02497038533371,1.54520953797025,-0.998749217771909,1614.31190318638,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"142",0.0362086635314198,0.452003571795249,1047.92215865736,-4.11677560596058,1.21197893345705,4.99374608885955,308.900677729789,0.276070082472555,"1" +"143",16.4091328365362,-0.723428206559773,-0.484837871827967,-2.2068409681768,583.664405744985,-4.02497038533371,0.00978544486714703,0.998749217771909,"1" +"144",114.85616004377,1.03953114431459,131.287665895645,-0.723877646928655,8.5439216236964,0.452003571795249,127.250500822684,-4.02497038533371,"1" +"145",-1.08720332527375,-1.04373592786303,358.550969180115,-0.959567210136544,155.765416281121,-0.723428206559773,3.05665029819286,5.76969265173935,"1" +"146",96.1704070107891,-1.1925100147406,40.6841720599538,-1.04373592786303,654.1807376406,1.03953114431459,237.265198814581,-0.722978766190891,"1" +"147",-0.702753949591203,-0.476696519712976,8160.44589674603,-1.1925100147406,56.7500856696567,-1.0360459908358,-2.99624765331573,-0.959567210136544,"1" +"148",122.984530765834,-0.00208617190785308,1.32646931591083,-0.476696519712976,444.479041747128,-0.702269515361112,-4.20858082658746,-1.04629924020543,"1" +"149",-0.889937187318475,0.998749217771909,6276.20926576642,-0.00208617190785308,-0.285673287268748,-0.452272463368005,258.359923949084,-0.702269515361112,"1" +"150",22.8292744548542,-4.11677560596058,26.1581346497832,-0.998749217771909,2449.93501534695,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"151",-0.134759730883379,0.452003571795249,1252.3524468986,-4.02497038533371,-0.62335270370792,2.99624765331573,652.745594695384,0.554226336852964,"1" +"152",19.7470078891011,-0.723428206559773,18.4112403870648,-2.2068409681768,107.150640348126,-3.93316516470684,-0.882247250291247,22.9712320087539,"1" +"153",66.9161975907179,1.03953114431459,241.437542630287,-0.723877646928655,13.168209625011,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"154",-0.53637200151251,-1.04373592786303,550.310818992322,-0.959567210136544,92.6239465367682,-0.722978766190891,-0.492979223942957,-2.2068409681768,"1" +"155",183.912276829867,-0.702269515361112,236.96373376021,-1.04629924020543,278.651031758363,3.03862949876572,-0.55839868066867,-0.723877646928655,"1" +"156",-0.703203389960085,-0.476696519712976,1422.93383245684,-0.702269515361112,64.7371398641947,-1.04117261552062,-2.99624765331573,-0.959567210136544,"1" +"157",96.996252157969,-0.00208617190785308,0.950737167525511,-0.484837871827967,2324.28213150737,-0.212029015981627,-4.20858082658746,-1.04629924020543,"1" +"158",-0.925823560112203,0.998749217771909,2331.98821243434,-0.00208617190785308,-0.496910260643267,-0.476696519712976,253.04223486914,1.74893298153631,"1" +"159",38.5169704349977,-4.02497038533371,1.91176320293476,-0.998749217771909,460.832152668075,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"160",-0.191749195688312,0.452003571795249,1993.59608196038,-4.11677560596058,0.278933240820111,0.998749217771909,-2.95866556458768,-0.280242426288262,"1" +"161",19.4688516347207,-0.723428206559773,35.23941520875,-2.2068409681768,438.062977429278,-3.93316516470684,-1.04886255254784,-0.998749217771909,"1" +"162",102.871169430507,1.03953114431459,-130.457369476319,-0.723877646928655,3.86264415757691,0.452003571795249,-1.68275051412008,-4.11677560596058,"1" +"163",0.19806976350248,-1.04373592786303,7341.8054998413,-0.959567210136544,157.434353807403,-0.723428206559773,0.288590579096123,5.76969265173935,"1" +"164",106.805785170677,-0.702269515361112,282.590928411766,-1.04373592786303,2401.99186874144,3.03862949876572,37.2708519150669,-0.722978766190891,"1" +"165",-0.69511346332021,-0.476696519712976,1561.19374853539,-1.1925100147406,29.4839351434752,-1.04373592786303,180.773608416716,3.03862949876572,"1" +"166",118.986334056931,-0.00208617190785308,-0.285673287268748,-0.484837871827967,803.423054643355,-0.702269515361112,68.5929591305234,-1.04373592786303,"1" +"167",-0.964273245248341,0.998749217771909,1988.14329546874,-0.280242426288262,-0.575112884828728,-0.476696519712976,261.018768489056,-0.212029015981627,"1" +"168",14.0049454660234,-4.02497038533371,-0.805347880018972,-0.998749217771909,1156.51838001707,0.276070082472555,-0.724327087297537,-0.484837871827967,"1" +"169",-0.0370635055034939,0.452003571795249,-185.032697282047,-4.11677560596058,1.04023700651564,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"170",10.0115389857869,-0.723428206559773,15.6187566116231,-2.2068409681768,163.038057277387,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"171",114.85616004377,1.03953114431459,929.317959713037,-0.723877646928655,2.31578725572874,11.0873817316835,-1.68275051412008,-4.11677560596058,"1" +"172",-0.903592884020004,-1.04373592786303,1219.4727948995,-0.959567210136544,84.5574151597363,-0.722978766190891,0.500265734085873,0.452003571795249,"1" +"173",96.1704070107891,-0.702269515361112,176.4640933671,-1.04629924020543,750.060662546704,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"174",-0.702304509222321,-0.484837871827967,218.477255849504,-0.702269515361112,96.9607723042273,-1.04373592786303,-2.99624765331573,-0.959567210136544,"1" +"175",71.0079735501043,-0.00208617190785308,-0.688821298155862,-0.476696519712976,7155.40266063659,3.21965447967477,-4.20858082658746,-1.04629924020543,"1" +"176",-0.925823560112203,0.998749217771909,2627.8547688931,-0.280242426288262,-0.576011765566492,-0.452272463368005,-4.86568550814886,-1.1925100147406,"1" +"177",15.9659074635414,-4.02497038533371,1.85793364374417,0.998749217771909,202.948464943879,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"178",-0.199890547803302,-2.2068409681768,662.593126145082,-4.02497038533371,0.840298643807721,2.99624765331573,-2.95866556458768,-0.280242426288262,"1" +"179",12.2367890208301,-0.723877646928655,-3.39130057687954,0.452003571795249,301.776118601781,-4.02497038533371,-0.136323358650176,2.99624765331573,"1" +"180",50.9362101063674,1.03953114431459,1611.63525170818,-0.723428206559773,8.42180134197154,3.1108481117673,-1.68275051412008,-4.11677560596058,"1" +"181",-0.995398104646878,-1.04373592786303,2148.30956742738,1.03953114431459,37.5490081694473,-0.719383243239835,-0.492979223942957,-2.2068409681768,"1" +"182",90.852717930845,-0.702269515361112,5.24735689798053,-1.04373592786303,2663.66416379768,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"183",-0.704102270697849,-0.476696519712976,747.587319303942,-1.1925100147406,60.146878832851,-1.04117261552062,306.616009855976,5.03772785321685,"1" +"184",106.991743930225,-0.00208617190785308,0.267587806824939,-0.484837871827967,689.092739424557,-0.702269515361112,4.23749947108492,-1.0360459908358,"1" +"185",-0.959146620563523,0.998749217771909,-0.959567210136544,-0.00208617190785308,-0.654663830120834,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"186",17.4366289616798,-4.02497038533371,-0.587466330914192,-0.998749217771909,632.754611150873,0.832382591233372,-0.724327087297537,-0.484837871827967,"1" +"187",-0.0696289139634555,0.452003571795249,731.717036557589,-4.11677560596058,3.94190657812283,6.99124452440336,-2.95866556458768,-0.280242426288262,"1" +"188",19.4688516347207,-0.723428206559773,4.83146505926077,-2.2068409681768,178.72575325753,-4.02497038533371,-1.04886255254784,-0.998749217771909,"1" +"189",92.8836772527875,1.03953114431459,435.868764442192,-0.723428206559773,2.67400674878831,0.452003571795249,127.250500822684,-3.47413906157247,"1" +"190",0.748901087263722,-1.04373592786303,3958.04315003008,1.03953114431459,44.2247582745771,-0.723428206559773,-0.492979223942957,-2.2068409681768,"1" +"191",128.076541490454,-1.1925100147406,153.604593431008,-1.04629924020543,420.473420681974,1.03953114431459,-0.55839868066867,-0.723877646928655,"1" +"192",-0.708596674386668,-0.476696519712976,978.906794281511,-1.1925100147406,75.5701558981658,-1.03860930317821,682.145715738214,7.03682620766799,"1" +"193",112.989038993578,-0.280242426288262,-0.512640673554136,-0.484837871827967,1274.03853821841,0.278211483397858,-4.20858082658746,-1.04629924020543,"1" +"194",-0.951456683536295,0.998749217771909,7441.68360641143,-0.280242426288262,-0.564775756344443,-0.476696519712976,-4.86568550814886,-1.1925100147406,"1" +"195",20.3780719579567,-4.02497038533371,18.9936766527495,-0.998749217771909,822.66895482373,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"1" +"196",-0.0696289139634555,0.452003571795249,1481.7850006082,-4.11677560596058,0.173837434781335,0.998749217771909,702.723053556662,-0.00208617190785308,"1" +"197",10.0115389857869,-0.723428206559773,1.05387767790522,-2.2068409681768,115.484728837577,-4.02497038533371,-0.772024819567653,2.99624765331573,"1" +"198",110.861163172682,1.03953114431459,2605.209392355,-0.723877646928655,2.12039480496897,3.1108481117673,48.3217804225874,-3.93316516470684,"1" +"199",0.0144593222487325,-1.04629924020543,1684.88993038121,-0.959567210136544,282.882824532968,-0.722529325822009,-0.492979223942957,-2.2068409681768,"1" +"200",165.300365050063,-0.702269515361112,354.290805721354,-1.04629924020543,718.100687578003,1.03953114431459,26.4227579942309,-0.722079885453127,"1" +"201",-0.710394435862196,-0.476696519712976,518.926688866346,-1.1925100147406,2.12597939666683,-1.04373592786303,200.748592772154,7.03682620766799,"0" +"202",67.009776841202,-0.00208617190785308,0.169609806408672,-0.476696519712976,138.711919650342,-0.702269515361112,20.3034130807878,-1.04117261552062,"0" +"203",-0.928386872454613,0.998749217771909,18628.63799792,-0.00208617190785308,-0.709944995493314,-0.427848407023034,-4.86568550814886,-1.1925100147406,"0" +"204",12.534223967885,-4.02497038533371,-1.02835605380857,0.998749217771909,450.83666089582,0.832382591233372,-0.724327087297537,-0.484837871827967,"0" +"205",-0.15104243511336,0.452003571795249,5115.93782250832,-4.02497038533371,-0.684872199925741,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"206",7.22997644198277,-0.723428206559773,7.2738706937579,0.452003571795249,165.489259774284,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"207",88.8886803816999,1.03953114431459,565.211422729082,-0.723877646928655,0.516548438315854,0.452003571795249,89.9922228698435,-3.74955472345309,"0" +"208",-0.81178766339313,-1.04373592786303,6245.17885872775,1.03953114431459,40.6087269676318,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"209",138.711919650342,-1.1925100147406,-80.6823296087732,-1.04373592786303,258.676047402924,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"210",-0.709495555124432,-0.476696519712976,566.785890585842,-0.702269515361112,13.3262163131454,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"211",73.0070719045554,-0.00208617190785308,0.0761262096812256,-0.476696519712976,542.856289726094,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"212",-0.961709932905932,0.998749217771909,3269.56534067192,-0.00208617190785308,-0.696012344057974,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"213",17.9268694610593,-4.02497038533371,8.45077298842057,0.998749217771909,152.971006082601,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"214",-0.15104243511336,0.452003571795249,-340.438935585344,-4.02497038533371,-0.725885197404287,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"215",13.9057265471126,-0.723428206559773,7.5262526093226,0.452003571795249,171.862386266217,-3.74955472345309,-1.04886255254784,-0.998749217771909,"0" +"216",104.86866786605,1.03953114431459,2399.3737641135,-0.723877646928655,0.0524913677614007,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"217",-1.82164509028874,-1.04373592786303,282.64602862945,1.03953114431459,86.5045089403992,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"218",93.5115624708171,-0.702269515361112,658.900527761321,-1.04373592786303,230.711069305311,7.03682620766799,42.5558207482946,-0.723428206559773,"0" +"219",-0.703203389960085,-0.476696519712976,5865.86305875014,-0.702269515361112,18.1918930063697,-1.03860930317821,-2.99624765331573,-0.959567210136544,"0" +"220",83.0025636768111,-0.00208617190785308,0.621297377135037,-0.476696519712976,359.396016468022,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"221",-0.933513497139431,0.998749217771909,6178.25344639832,-0.00208617190785308,-0.575112884828728,-0.452272463368005,-4.86568550814886,-1.1925100147406,"0" +"222",17.9268694610593,-4.02497038533371,-0.331135096673274,0.998749217771909,720.714938746722,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"223",-0.191749195688312,0.452003571795249,278.734815130945,-4.02497038533371,-0.928386872454613,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"224",12.7931015295909,-0.723428206559773,27.6842404460389,0.452003571795249,191.962246740776,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"225",68.9136960262617,1.03953114431459,336.845137882767,-0.723428206559773,1.42837987519478,0.452003571795249,171.372145766838,-4.02497038533371,"0" +"226",-1.2708137665275,-1.04373592786303,12259.6466481502,-0.959567210136544,66.7554148793902,-0.721630445084245,-0.492979223942957,-2.2068409681768,"0" +"227",106.805785170677,-1.1925100147406,-38.6355385616651,-1.04373592786303,839.948092146176,5.03772785321685,65.9209461162489,-0.719832683608717,"0" +"228",-0.708147234017786,-0.476696519712976,9043.18228401675,-1.1925100147406,20.5788287426684,-1.04373592786303,386.515947277729,3.03862949876572,"0" +"229",100.994448866871,-0.00208617190785308,-0.665450398974001,-0.484837871827967,1452.18112239654,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"230",-0.961709932905932,0.998749217771909,464.830349376977,-0.00208617190785308,-0.531517169047178,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"231",20.3780719579567,-4.02497038533371,0.819792145068448,0.998749217771909,41.0214982333372,2.77947637189623,-0.724327087297537,-0.484837871827967,"0" +"232",0.00364325507145819,0.452003571795249,409.629028465268,-4.11677560596058,-0.669492325871285,0.998749217771909,-2.95866556458768,-0.280242426288262,"0" +"233",8.62075771388481,-0.723428206559773,-0.75350249162265,0.452003571795249,113.523766840059,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"234",74.9061913328932,1.03953114431459,71.4840712038571,-0.723877646928655,0.988746860985298,8.42853719171141,-1.68275051412008,-4.11677560596058,"0" +"235",0.106264542875606,-1.04629924020543,2.99624765331573,1.03953114431459,51.7349771428481,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"236",88.193873390873,-0.702269515361112,36.185716249237,-1.04373592786303,692.133207915933,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"237",-0.705001151435613,-0.476696519712976,-2.2068409681768,-0.702269515361112,2.49320027917432,-1.04373592786303,306.616009855976,3.03862949876572,"0" +"238",73.0070719045554,-0.00208617190785308,-0.0978072130760911,-0.484837871827967,340.784104688218,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"239",-0.920696935427385,-0.998749217771909,59.0133834233974,-0.280242426288262,-0.526573324989476,-0.468555167597986,-4.86568550814886,-1.1925100147406,"0" +"240",14.4951859654029,-4.02497038533371,2.11170156564267,-0.998749217771909,192.952973171623,0.832382591233372,-0.724327087297537,-0.484837871827967,"0" +"241",-0.183607843573321,-2.2068409681768,17.9268694610593,-4.11677560596058,-0.856614126867156,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"242",15.0183515646342,-0.723428206559773,9.38248089154041,-2.2068409681768,116.955450335715,-4.02497038533371,-0.81047450470379,0.998749217771909,"0" +"243",82.8961850750684,1.03953114431459,131.287665895645,-0.723428206559773,1.27369418500996,11.0873817316835,-1.68275051412008,-4.11677560596058,"0" +"244",-1.08720332527375,-1.04373592786303,11266.8899256849,-0.959567210136544,46.4500083096204,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"245",154.664986890174,-0.702269515361112,-12.3792454623792,-1.04629924020543,80.8986866395246,9.03592456211912,-0.55839868066867,-0.723877646928655,"0" +"246",-0.705900032173377,-0.476696519712976,5190.51654559724,-0.702269515361112,6.16540910424927,-1.03348267849339,-2.99624765331573,-0.959567210136544,"0" +"247",63.0115801322997,-0.280242426288262,-0.723877646928655,-0.476696519712976,614.645092305339,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"248",-0.910443686057748,-0.998749217771909,1116.53641292805,-0.280242426288262,-0.716686601026544,-0.444131111253015,-4.86568550814886,-1.1925100147406,"0" +"249",23.8097554536131,-4.11677560596058,4.90827533121109,0.998749217771909,300.904284311985,2.50132011751582,-0.724327087297537,-0.484837871827967,"0" +"250",-0.224314604148273,0.452003571795249,64.0094764027309,-4.02497038533371,0.0328552559488296,0.998749217771909,-2.95866556458768,-0.280242426288262,"0" +"251",11.1241640033085,-0.723428206559773,21.635215824601,0.452003571795249,272.851929138391,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"252",84.8936835106123,-0.959567210136544,342.964575479136,-0.723428206559773,2.01455722747409,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"253",-1.54622942840812,-1.04373592786303,1602.99249452391,1.03953114431459,89.8423839929641,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"254",77.5584952309848,-0.702269515361112,63.7272824372991,-1.04629924020543,324.59349577587,3.03862949876572,-0.55839868066867,-0.723877646928655,"0" +"255",-0.706349472542259,-0.476696519712976,2919.86330846111,-1.1925100147406,15.2541259463098,-1.04117261552062,488.388367490464,11.0350229165703,"0" +"256",71.0079735501043,-0.00208617190785308,-0.605225389543819,-0.476696519712976,1279.35622729835,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"257",-0.900190436688112,0.998749217771909,3217.58878345619,-0.280242426288262,-0.657809912703007,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"258",15.9659074635414,-4.11677560596058,-0.777151444252471,0.998749217771909,234.934038615097,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"259",-0.0126394491585226,0.452003571795249,-0.212029015981627,-4.02497038533371,-0.323445159646046,16.9787367021225,-2.95866556458768,-0.280242426288262,"0" +"260",9.17707022264563,-0.723428206559773,-0.484837871827967,-2.2068409681768,58.6168309095565,-4.02497038533371,-0.861740751551974,0.998749217771909,"0" +"261",90.8861788172437,-0.959567210136544,5.83919517008072,-0.723877646928655,-0.419707054908043,24.3816044315437,-1.68275051412008,-4.11677560596058,"0" +"262",1.11612196977122,-1.04373592786303,13975.4978042823,-0.959567210136544,32.2640393362195,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"263",77.5584952309848,-0.702269515361112,1.57514807290558,-1.04629924020543,424.468417553061,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"264",-0.699158426640147,-0.476696519712976,5073.52738583847,-1.1925100147406,11.2146962387273,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"265",104.992645575774,-0.00208617190785308,2.48153106393745,-0.484837871827967,231.771478549364,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"266",-0.954019995878704,0.998749217771909,1904.18116458179,-0.00208617190785308,-0.564775756344443,-0.468555167597986,888.506079922461,0.278211483397858,"0" +"267",13.5147049666439,-4.02497038533371,2.38341267393805,0.998749217771909,160.967399500405,1.38869509999419,-0.724327087297537,-0.484837871827967,"0" +"268",-0.0126394491585226,0.452003571795249,713.087897581169,-4.02497038533371,-0.677182262898513,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"269",15.8528203277754,-0.723428206559773,38.9193063647256,-2.2068409681768,178.235512758151,-4.02497038533371,-1.04886255254784,-0.998749217771909,"0" +"270",112.858661608226,1.03953114431459,292.340137181901,-0.723428206559773,1.36324905827486,8.42853719171141,-1.68275051412008,-4.11677560596058,"0" +"271",-0.719982442766257,-1.04629924020543,-0.998749217771909,1.03953114431459,4.44841389817868,-0.720731564346481,-0.492979223942957,-2.2068409681768,"0" +"272",90.852717930845,-0.702269515361112,12.5917745481304,-1.04629924020543,756.053157853335,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"273",-0.700506747746793,-0.484837871827967,1319.23889539793,-1.1925100147406,21.3132705076834,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"274",102.993547221322,-0.280242426288262,-0.632191811676736,-0.484837871827967,354.078327388078,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"275",-0.948893371193886,0.998749217771909,1986.14419711429,-0.280242426288262,-0.670394243031702,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"276",17.4366289616798,-4.02497038533371,5.48502060825315,0.998749217771909,118.986334056931,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"277",-0.0696289139634555,-2.2068409681768,158.135652283592,-4.02497038533371,-0.395217905233503,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"278",10.5678514945477,-0.723877646928655,2.62515863609837,-2.2068409681768,97.3458303605358,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"279",106.866166301594,1.03953114431459,1102.60930619203,-0.723428206559773,0.63866872004071,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"280",-1.08720332527375,-1.04373592786303,-0.998749217771909,-0.959567210136544,58.4107272479779,-0.722529325822009,-0.492979223942957,-2.2068409681768,"0" +"281",80.2173397709568,-0.702269515361112,56.8418908902835,-1.04373592786303,1970.53220666398,5.03772785321685,-0.55839868066867,-0.723877646928655,"0" +"282",-0.704551711066731,-0.476696519712976,239.74801216928,-0.702269515361112,-0.719982442766257,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"283",77.0052686134576,-0.00208617190785308,-0.714888839551016,-0.476696519712976,287.607213888777,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"284",-0.913006998400158,-0.998749217771909,852.655430140497,-0.280242426288262,-0.707697793648904,-0.460413815482995,-4.86568550814886,-1.1925100147406,"0" +"285",23.8097554536131,-4.02497038533371,6.4744591724231,0.998749217771909,1452.38493647584,0.832382591233372,-0.724327087297537,-0.484837871827967,"0" +"286",-0.126618378768388,-2.2068409681768,-1.1925100147406,-4.11677560596058,-0.866867376236792,18.9762351376663,-2.95866556458768,-0.280242426288262,"0" +"287",13.0712577839713,-0.723428206559773,7.7786345248873,-2.2068409681768,47.8315399232079,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"288",122.846153785945,-0.959567210136544,450.054733415593,-0.723877646928655,3.86264415757691,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"289",-0.169151119005015,-1.04629924020543,48.9387116708235,1.03953114431459,65.3646336074881,-0.720731564346481,-0.492979223942957,-2.2068409681768,"0" +"290",122.75885241051,-0.702269515361112,72.9996097206133,-1.04629924020543,1015.72795447403,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"291",-0.698259545902383,-0.476696519712976,20949.4881340116,-1.1925100147406,25.0772845533852,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"292",67.009776841202,-0.00208617190785308,-0.747248546110516,-0.484837871827967,29.6992935114878,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"293",-0.943766746509068,-0.998749217771909,-0.959567210136544,-0.280242426288262,-0.693765142213564,-0.468555167597986,-4.86568550814886,-1.1925100147406,"0" +"294",20.8683124573362,-4.02497038533371,-0.928386872454613,0.998749217771909,466.829447731429,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"295",-0.0289221533885035,0.452003571795249,1542.57482253126,-4.11677560596058,-0.682308887583331,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"296",11.1241640033085,-0.723428206559773,14.837186808584,0.452003571795249,79.6971723828744,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"297",92.8836772527875,1.03953114431459,41.7213519851534,-0.723428206559773,1.06201903002021,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"298",-1.08720332527375,-1.04373592786303,985.765477940874,1.03953114431459,43.9466020201967,-0.723428206559773,2.33206995995872,3.1108481117673,"0" +"299",114.782318790594,-1.1925100147406,31.5036499972664,-1.04373592786303,490.385865926007,3.03862949876572,102.637571694463,-0.722978766190891,"0" +"300",-0.701405628484557,-0.484837871827967,354.078327388078,-1.1925100147406,20.0279974189072,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"301",132.980022538089,-0.280242426288262,0.0608452371392392,-0.484837871827967,354.078327388078,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"302",-0.938640121824249,0.998749217771909,29.0269081066305,-0.00208617190785308,-0.620506362085805,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"303",18.4171099604388,-4.02497038533371,-0.774588131910062,0.998749217771909,398.86010368009,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"304",-0.126618378768388,0.452003571795249,541.01348229897,-4.02497038533371,-0.590029643256601,8.98874295994718,236.933136969548,2.77947637189623,"0" +"305",15.0183515646342,-0.723428206559773,0.809637114455509,0.452003571795249,89.0117418710846,-4.02497038533371,-1.04886255254784,-0.998749217771909,"0" +"306",114.85616004377,1.03953114431459,37.5490081694473,-0.723428206559773,1.13529119905513,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"307",0.840706307890596,-1.04373592786303,152.808630319102,-0.959567210136544,49.7878833621853,-0.722079885453127,-0.492979223942957,-2.2068409681768,"0" +"308",120.100007870538,-0.702269515361112,12.0409432243692,-1.04629924020543,260.673545838468,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"309",-0.707697793648904,-0.476696519712976,58.9465834511804,-0.702269515361112,9.83761792932422,-1.04117261552062,692.133207915933,1.03953114431459,"0" +"310",73.0070719045554,-0.00208617190785308,-0.623652444667979,-0.484837871827967,61.6054279911525,3.70989497905425,-4.20858082658746,-1.04629924020543,"0" +"311",-0.964273245248341,0.998749217771909,5324.63844904768,-0.00208617190785308,-0.615113077659222,-0.427848407023034,-4.86568550814886,-1.1925100147406,"0" +"312",17.9268694610593,-4.02497038533371,3.04218394593721,0.998749217771909,202.948464943879,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"313",-0.16732513934334,0.452003571795249,703.773328092958,-4.02497038533371,-0.502877023614689,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"314",16.1309765821558,-0.723428206559773,8.13685401794688,-2.2068409681768,49.3022614213463,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"315",72.9086928973494,1.03953114431459,746.569300585108,-0.723877646928655,1.03759497367524,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"316",-1.2708137665275,-1.04373592786303,2653.67667161996,1.03953114431459,36.9926956606865,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"317",114.782318790594,-0.702269515361112,39.5825094124313,-1.04373592786303,943.818010794454,1.03953114431459,-0.55839868066867,-0.723877646928655,"0" +"318",-0.706349472542259,-0.476696519712976,2055.73883297019,-0.702269515361112,-1.45442420778125,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"319",96.996252157969,-0.00208617190785308,-0.452865104492836,-0.476696519712976,1566.51143761533,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"320",-0.925823560112203,0.998749217771909,1832.21362382155,-0.280242426288262,-0.491067535847802,-0.460413815482995,561.468201505898,1.25869248215683,"0" +"321",20.3780719579567,-4.02497038533371,4.12390175443388,0.998749217771909,146.973711019247,0.554226336852964,-0.724327087297537,-0.484837871827967,"0" +"322",-0.110335674538408,0.452003571795249,6.16109747595167,-4.11677560596058,-0.487497149560234,8.98874295994718,-2.95866556458768,-0.280242426288262,"0" +"323",14.7401953102538,-0.723877646928655,21.9934353176606,-2.2068409681768,376.782915006842,-4.02497038533371,-1.04886255254784,-0.998749217771909,"0" +"324",50.9362101063674,1.03953114431459,793.855863829778,-0.723877646928655,-0.0614875618484651,5.76969265173935,-1.68275051412008,-4.11677560596058,"0" +"325",-0.995398104646878,-1.04373592786303,827.963101532913,-0.959567210136544,100.134165405039,-0.722978766190891,-0.492979223942957,-2.2068409681768,"0" +"326",149.34729781023,-0.702269515361112,155.716113505426,-1.04629924020543,726.090681320178,3.03862949876572,-0.55839868066867,-0.723877646928655,"0" +"327",-0.707248353280022,-0.476696519712976,-2.2068409681768,-0.702269515361112,20.8542444045491,-1.04373592786303,-2.99624765331573,-0.959567210136544,"0" +"328",87.0007603857133,-0.00208617190785308,-0.643877261267667,-0.484837871827967,332.807571068302,-0.212029015981627,-4.20858082658746,-1.04629924020543,"0" +"329",-0.913006998400158,0.998749217771909,-980.517760891192,-0.00208617190785308,-0.69061905963139,-0.403424350678062,-4.86568550814886,-1.1925100147406,"0" +"330",15.9659074635414,-4.11677560596058,-2.88675750205523,0.998749217771909,92.9980554490667,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"331",-0.0452048576184843,-2.2068409681768,3474.12239008643,-4.11677560596058,-0.943766746509068,0.998749217771909,-2.95866556458768,-0.280242426288262,"0" +"332",9.45522647702603,-0.723428206559773,-0.484837871827967,-2.2068409681768,236.08389168493,-3.74955472345309,-1.04886255254784,-0.998749217771909,"0" +"333",60.9237022840865,1.03953114431459,389.138513706284,-0.723428206559773,0.939898748295356,3.1108481117673,-0.212029015981627,-4.02497038533371,"0" +"334",-0.903592884020004,-1.04373592786303,2322.09193131969,1.03953114431459,78.7161338177477,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"335",93.5115624708171,-0.702269515361112,80.7112482532707,-1.04373592786303,344.568480131309,7.03682620766799,-0.55839868066867,-0.723877646928655,"0" +"336",-0.700057307377911,-0.476696519712976,-967.367408978032,-0.702269515361112,14.336073740041,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"337",87.0007603857133,-0.00208617190785308,-0.600730985854999,-0.476696519712976,1085.26057588039,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"338",-0.920696935427385,0.998749217771909,9420.79097731805,-0.00208617190785308,-0.70904611475555,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"339",28.712160447408,-4.02497038533371,-0.597719580283828,-0.998749217771909,156.969202791503,0.276070082472555,-0.641630059423257,-0.468555167597986,"0" +"340",-0.0533462097334747,0.452003571795249,99.7970328574332,-4.11677560596058,-0.823291066415836,10.986241395491,-2.95866556458768,-0.280242426288262,"0" +"341",8.06444520512399,-0.723428206559773,5.09198832694047,0.452003571795249,11.0635024697465,-3.84135994407996,-1.04886255254784,-0.998749217771909,"0" +"342",62.9212007196303,1.03953114431459,-127.675806932515,-0.723877646928655,0.679375480615662,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"343",-0.0773458983781412,-1.04373592786303,27670.3470783707,1.03953114431459,369.389419645275,-0.722529325822009,-0.492979223942957,-2.2068409681768,"0" +"344",112.123474250621,-1.1925100147406,111.5578023839,-1.04629924020543,32.958724186473,7.03682620766799,-0.55839868066867,-0.723877646928655,"0" +"345",-0.707248353280022,-0.476696519712976,1252.76778189863,-0.702269515361112,22.8739592583403,-1.03860930317821,-2.99624765331573,-0.959567210136544,"0" +"346",106.991743930225,-0.00208617190785308,-0.886125620095041,-0.476696519712976,601.350869605479,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"347",-0.964273245248341,0.998749217771909,5102.73853170361,-0.00208617190785308,-0.517584517611837,-0.427848407023034,-4.86568550814886,-1.1925100147406,"0" +"348",20.3780719579567,-4.11677560596058,0.332762800010704,-0.998749217771909,390.863710262286,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"349",-0.142901082998369,0.452003571795249,4.20013547843373,-4.11677560596058,-0.761771570198016,4.99374608885955,-2.95866556458768,-0.280242426288262,"0" +"350",11.4023202576889,-0.723428206559773,1.7540339597944,0.452003571795249,66.9509193990078,-3.56594428219934,-1.04886255254784,-0.998749217771909,"0" +"351",78.9011882039808,1.03953114431459,156.321728789882,-0.723428206559773,4.52209367889114,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"352",-1.17900854590063,-1.04373592786303,3135.07379458602,-0.959567210136544,43.1121332570555,-0.723428206559773,-0.492979223942957,-2.2068409681768,"0" +"353",149.34729781023,-0.702269515361112,455.827379734677,-1.04629924020543,1137.5753590422,7.03682620766799,-0.55839868066867,-0.723877646928655,"0" +"354",-0.704551711066731,-0.484837871827967,420.54944088738,-1.1925100147406,6.16540910424927,-1.0360459908358,-2.99624765331573,-0.959567210136544,"0" +"355",81.0034653223599,-0.00208617190785308,-0.648371664956486,-0.476696519712976,494.997088006597,0.278211483397858,-4.20858082658746,-1.04629924020543,"0" +"356",-0.895063812003293,0.998749217771909,1422.39846115907,-0.00208617190785308,-0.674888646720522,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"357",22.3390339554747,-4.02497038533371,2.9575946386377,-0.998749217771909,98.9953505124201,0.276070082472555,-0.724327087297537,-0.484837871827967,"0" +"358",-0.102194322423417,0.452003571795249,974.386083750434,-4.02497038533371,-0.769461507225243,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"359",8.89891396826522,-0.723428206559773,-0.484837871827967,0.452003571795249,20.3780719579567,-3.38233384094559,-1.04886255254784,-0.998749217771909,"0" +"360",80.8986866395246,1.03953114431459,-4.45258624199439,-0.723428206559773,0.891050635605413,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"361",-0.903592884020004,-1.04373592786303,1421.22013688943,1.03953114431459,73.7093212389004,-0.722079885453127,-0.492979223942957,-2.2068409681768,"0" +"362",133.394230570398,-1.1925100147406,-0.0773458983781412,-1.04629924020543,272.658536451731,3.03862949876572,-0.55839868066867,-0.723877646928655,"0" +"363",-0.705900032173377,-0.476696519712976,819.376121883188,-0.702269515361112,16.1721781525785,-1.04117261552062,-2.99624765331573,-0.959567210136544,"0" +"364",75.0061702590065,-0.00208617190785308,-0.741855261683933,-0.476696519712976,601.350869605479,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"365",-0.90531706137293,0.998749217771909,1904.18116458179,-0.00208617190785308,-0.66769760081841,-0.460413815482995,-4.86568550814886,-1.1925100147406,"0" +"366",17.9268694610593,-4.11677560596058,14.0080341467637,-0.998749217771909,558.787972036181,-0.00208617190785308,-0.569270160033262,-0.468555167597986,"0" +"367",-0.126618378768388,0.452003571795249,30.6731224449259,-4.11677560596058,-0.910443686057748,20.9737335732101,-2.95866556458768,-0.280242426288262,"0" +"368",10.2896952401673,-0.723428206559773,54.1110694112977,-2.2068409681768,68.9118813965257,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"369",84.8936835106123,-0.959567210136544,316.261575058617,-0.723877646928655,0.0443500156464103,0.452003571795249,-1.68275051412008,-4.11677560596058,"0" +"370",0.473485425383101,-1.04373592786303,1772.77986154514,-0.959567210136544,44.2247582745771,-0.722529325822009,-0.492979223942957,-2.2068409681768,"0" +"371",104.146940630705,-0.702269515361112,80.8030534738976,-1.04373592786303,42.9462163641921,15.0332196254725,-0.55839868066867,-0.723877646928655,"0" +"372",-0.699607867009029,-0.476696519712976,258.359923949084,-1.1925100147406,34.4414170573264,-1.0360459908358,-2.99624765331573,-0.959567210136544,"0" +"373",87.0007603857133,-0.00208617190785308,-0.120728671889071,-0.476696519712976,295.583747508693,-0.212029015981627,29.8511560259827,-1.04117261552062,"0" +"374",-0.910443686057748,0.998749217771909,270.917808995218,-0.280242426288262,-0.609719793232638,-0.476696519712976,-4.86568550814886,-1.1925100147406,"0" +"375",16.4561479629209,-4.02497038533371,3.05756381999166,0.998749217771909,324.893464565398,0.554226336852964,-0.724327087297537,-0.484837871827967,"0" +"376",-0.0452048576184843,0.452003571795249,416.002154957201,-4.02497038533371,-0.769461507225243,2.99624765331573,-2.95866556458768,-0.280242426288262,"0" +"377",10.8460077489281,-0.723428206559773,-0.484837871827967,0.452003571795249,80.6776533816333,-3.93316516470684,-1.04886255254784,-0.998749217771909,"0" +"378",104.86866786605,1.03953114431459,-21.4201177591993,-0.723877646928655,0.923616044065375,0.452003571795249,79.2069318834949,-3.74955472345309,"0" +"379",0.473485425383101,-1.04373592786303,3093.1263274396,1.03953114431459,123.499290772993,-0.722978766190891,1.94942641055417,13.7462262716555,"0" +"380",130.735386030426,-0.702269515361112,168.293428731308,-1.04373592786303,88.8886803816999,3.03862949876572,50.0660396165657,-0.718933802870953,"0" +"381",-0.709495555124432,-0.476696519712976,-672.235665041134,-0.702269515361112,0.657095866636848,-1.0360459908358,-2.99624765331573,-0.959567210136544,"0" +"382",94.9971538035178,-0.00208617190785308,2.19254090674635,-0.476696519712976,697.069273044473,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" +"383",-0.930950184797022,0.998749217771909,4692.92336904112,-0.00208617190785308,-0.502752985438733,-0.476696519712976,391.302150947687,-0.702269515361112,"0" +"384",20.3780719579567,-4.02497038533371,8.34824049472421,-0.998749217771909,648.747397986482,-0.00208617190785308,-0.724327087297537,-0.484837871827967,"0" +"385",-0.118477026653398,-2.2068409681768,10.0830214709875,-4.02497038533371,-0.887373874976066,6.99124452440336,-2.95866556458768,-0.280242426288262,"0" +"386",7.78628895074358,-0.723428206559773,-1.41295201293687,0.452003571795249,63.5192359033514,-3.93316516470684,-0.126070109280539,2.99624765331573,"0" +"387",64.9186991551741,1.03953114431459,99.299696641898,-0.723877646928655,1.39581446673482,0.452003571795249,170.391664768079,-4.02497038533371,"0" +"388",-0.81178766339313,-1.04373592786303,2735.57410747726,-0.959567210136544,61.192289791782,-0.721630445084245,-0.492979223942957,-2.2068409681768,"0" +"389",96.1704070107891,-0.702269515361112,12.0409432243692,-1.04373592786303,292.633520807169,3.03862949876572,96.2399778437135,-0.723428206559773,"0" +"390",-0.705450591804495,-0.476696519712976,-2.2068409681768,-1.1925100147406,73.917661926882,-1.03860930317821,-2.99624765331573,-0.959567210136544,"0" +"391",61.0124817778486,-0.280242426288262,0.0702834848857602,-0.484837871827967,1048.03675232078,1.74893298153631,-4.20858082658746,-1.04629924020543,"0" +"392",-0.900190436688112,-0.998749217771909,1818.21993534039,-0.00208617190785308,-0.641630059423257,-0.468555167597986,-4.86568550814886,-1.1925100147406,"0" +"393",21.8487934560952,-4.11677560596058,-1.17190154498348,0.998749217771909,27.0278097521793,1.38869509999419,-0.576910646304255,-0.297586773183187,"0" +"394",-0.142901082998369,0.452003571795249,121.367614830131,-4.02497038533371,-0.928386872454613,30.9612257509292,-2.95866556458768,-0.280242426288262,"0" +"395",9.17707022264563,-0.723428206559773,19.7871288944982,0.452003571795249,89.5019823704641,-3.93316516470684,-0.584903018571783,0.998749217771909,"0" +"396",102.871169430507,1.03953114431459,228.086042420027,-0.723428206559773,2.98337812915795,3.1108481117673,-1.68275051412008,-4.11677560596058,"0" +"397",-0.444566780885636,-1.04373592786303,7375.76297324555,1.03953114431459,15.574664073395,-0.719383243239835,-0.492979223942957,-2.2068409681768,"0" +"398",93.5115624708171,-0.702269515361112,267.351261787705,-1.04373592786303,384.518448842185,3.03862949876572,73.43116498452,-0.722529325822009,"0" +"399",-0.700506747746793,-0.484837871827967,526.903222486262,-0.702269515361112,30.1265716878633,-1.04373592786303,470.410881570569,3.03862949876572,"0" +"400",77.0052686134576,-0.00208617190785308,-0.455112306337246,-0.476696519712976,1178.32013477941,-0.702269515361112,-4.20858082658746,-1.04629924020543,"0" diff --git a/example/HamiltonianVI/hamiltonian_layer.jl b/example/HamiltonianVI/hamiltonian_layer.jl index 9194be52..aabb4afe 100644 --- a/example/HamiltonianVI/hamiltonian_layer.jl +++ b/example/HamiltonianVI/hamiltonian_layer.jl @@ -1,4 +1,5 @@ using Functors +using Flux using Bijectors using Bijectors: partition, combine, PartitionMask using SimpleUnPack: @unpack @@ -17,8 +18,8 @@ struct LeapFrog{T<:Real,I<:Int} <: Bijectors.Bijector end @functor LeapFrog (ϵ,) -function LeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇logp, ∇logm) - return LeapFrog(dim, ϵ .* one.(ϵ), L, ∇logp, ∇logm) +function LeapFrog(dim::Int, ϵ::T, L::Int, ∇logp, ∇logm) where {T<:Real} + return LeapFrog(dim, ϵ .* ones(T, dim), L, ∇logp, ∇logm) end function Bijectors.inverse(lf::LeapFrog) @@ -27,24 +28,23 @@ function Bijectors.inverse(lf::LeapFrog) end function Bijectors.transform(lf::LeapFrog, z::AbstractVector) - @unpack d, ϵ, L, ∇logp, ∇logm = lf - @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" - mask = PartitionMask(n, 1:(lf.dim)) - x, ρ, emp = partition(mask, z) - - ρ += ϵ / 2 .* ∇logp(x) - for i in 1:(lf.L - 1) + @unpack dim, ϵ, L, ∇logp, ∇logm = lf + @assert length(z) == 2dim "dimension of input must be even, z = [x, ρ]" + # mask = PartitionMask(n, 1:dim) + # x, ρ, emp = partition(mask, z) + x, ρ = z[1:dim], z[(dim + 1):end] + + ρ += ϵ ./ 2 .* ∇logp(x) + for i in 1:(L - 1) x -= ϵ .* ∇logm(ρ) ρ += ϵ .* ∇logp(x) end x -= ϵ .* ∇logm(ρ) - ρ += ϵ / 2 .* ∇logp(x) - return combine(mask, x, ρ, emp) + ρ += ϵ ./ 2 .* ∇logp(x) + # return combine(mask, x, ρ, emp) + return vcat(x, ρ) end -# leapfrog composes shear transformations, hence has unit jacobian -Bijector.logabsdetjac(lf::LeapFrog, z::AbstractVector) = zero(eltype(z)) - function Bijectors.with_logabsdet_jacobian(lf::LeapFrog, z::AbstractVector) return Bijectors.transform(lf, z), zero(eltype(z)) end @@ -53,10 +53,14 @@ abstract type TrainableScore end struct CoresetScore{T<:AbstractVector} <: TrainableScore "coreset weights" w::T - "weighted coreset score function of the target, ∇logpw(w, x)" + "weighted coreset score function of the target, ∇logpw(x, w)" ∇logpw end @functor CoresetScore (w,) +function CoresetScore(T, coresize::Int, datasize::Int, ∇logpw) + return CoresetScore(ones(T, coresize) .* N ./ coresize, ∇logpw) +end +(C::CoresetScore)(x::AbstractVector) = C.∇logpw(x, C.w) struct SurrogateLeapFrog{T<:Real,I<:Int,H<:Union{TrainableScore,Flux.Chain}} <: Bijectors.Bijector @@ -74,34 +78,35 @@ end @functor SurrogateLeapFrog (ϵ, ∇S) -function SurrogateLeapFrog(dim::Int, ϵ::AbstractVector{T}, L::Int, ∇S, ∇logm) - return SurrogateLeapFrog(dim, ϵ .* one.(ϵ), L, ∇S, ∇logm) +function SurrogateLeapFrog(dim::Int, ϵ::T, L::Int, ∇S, ∇logm) where {T<:Real} + return SurrogateLeapFrog(dim, ϵ .* ones(T, dims), L, ∇S, ∇logm) end -function Bijectors.inverse(slf::SurrogateLeapFrogeapFrog) - @unpack d, ϵ, L, ∇S, ∇logm = slf - return SurrogateLeapFrog(d, -ϵ, L, ∇S, ∇logm) +function Bijectors.inverse(slf::SurrogateLeapFrog) + @unpack dim, ϵ, L, ∇S, ∇logm = slf + return SurrogateLeapFrog(dim, -ϵ, L, ∇S, ∇logm) end function Bijectors.transform(slf::SurrogateLeapFrog, z::AbstractVector) - @unpack d, ϵ, L, ∇S, ∇logm = slf - @assert length(z) == 2d "dimension of input must be even, z = [x, ρ]" - mask = PartitionMask(n, 1:(lf.dim)) - x, ρ, emp = partition(mask, z) - - ρ += ϵ / 2 .* ∇S(x) - for i in 1:(lf.L - 1) + @unpack dim, ϵ, L, ∇S, ∇logm = slf + n = length(z) + @assert n == 2dim "dimension of input must be even, z = [x, ρ]" + # mask = PartitionMask(n, 1:dim) + x, ρ = z[1:dim], z[(dim + 1):end] + # x, ρ, emp = partition(mask, z) + + ρ += ϵ ./ 2 .* ∇S(x) + for i in 1:(L - 1) x -= ϵ .* ∇logm(ρ) ρ += ϵ .* ∇S(x) end x -= ϵ .* ∇logm(ρ) - ρ += ϵ / 2 .* ∇S(x) - return combine(mask, x, ρ, emp) + ρ += ϵ ./ 2 .* ∇S(x) + # return combine(mask, x, ρ, emp) + return vcat(x, ρ) end # leapfrog composes shear transformations, hence has unit jacobian -Bijector.logabsdetjac(slf::SurrogateLeapFrog, z::AbstractVector) = zero(eltype(z)) - function Bijectors.with_logabsdet_jacobian(slf::SurrogateLeapFrog, z::AbstractVector) return Bijectors.transform(slf, z), zero(eltype(z)) end diff --git a/example/HamiltonianVI/main.jl b/example/HamiltonianVI/main.jl new file mode 100644 index 00000000..ca83c093 --- /dev/null +++ b/example/HamiltonianVI/main.jl @@ -0,0 +1,161 @@ +using CSV, DataFrames +using Random, Distributions, LinearAlgebra, Bijectors +using ADTypes +using Optimisers +using Tullio: @tullio +using FunctionChains +using NormalizingFlows +using Zygote +using Zygote: @adjoint, Buffer +using Bijectors: Shift, Scale +include("../common.jl") +include("hamiltonian_layer.jl") + +##############################33 +# model for posterior inference (logistic regression) +################################# + +######################################################################################### +# Example of Bayesian Logistic Regression (the same setting as Gershman et al. 2012): +# The observed data D = {X, y} consist of N binary class labels, +# y_t \in {-1,+1}, and d covariates for each datapoint, X_t \in R^d. +# The hidden variables \theta = {w, \alpha} consist of d regression coefficients w_k \in R, +# and a precision parameter \alpha \in R_+. We assume the following model: +# p(α) = Gamma(α ; a, b) , τ = log α ∈ R +# p(w_k | τ) = N(w_k; 0, exp(-τ)) +# p(y_t = 1| x_t, w) = 1 / (1+exp(-w^T x_t)), y ∈ {1, 0} +######################################################################################### +df = DataFrame(CSV.File("example/HamiltonianVI/data/bank_dat.csv")) +xs = Matrix(df)[:, 2:end] +X_raw = xs[:, 1:(end - 1)] +const X = (X_raw .- mean(X_raw; dims=1)) ./ std(X_raw; dims=1) +const Y = xs[:, end] +const a, b = 1.0, 0.01 +(N, p) = size(X) +idx = sample(1:N, 20; replace=false) +const Xc, Yc = X[idx, :], Y[idx] + +function log_sigmoid(x) + if x < -300 + return x + else + return -log1p(exp(-x)) + end +end + +function neg_sigmoid(x) + return -1.0 / (1.0 + exp(-x)) +end + +# z = (τ, w1, ..., wd) +function logp(θ, X, Y, w) + τ = θ[1] + W = @view(θ[2:end]) + Z = X * W + logpτ = a * τ - b * exp(τ) + logpW = 0.5 * p * τ - 0.5 * exp(τ) * sum(abs2, W) + @tullio llh := w[n] * ((Y[n] - 1.0) * Z[n] + log_sigmoid(Z[n])) + # llh = sum((Y .- 1.) .* Z .- log1p.(exp.(-Z))) + return logpτ + logpW + llh +end + +function logp_subsample(θ; batch_size=10) + (N, p) = size(X) + idx = sample(1:N, batch_size; replace=false) + w = N / batch_size .* ones(batch_size) + return logp(θ, X[idx, :], Y[idx], w) +end + +function ∇logp(z, X, Y, w) + τ = z[1] + W = @view(z[2:end]) + grad = similar(z) + grad[1] = a - b * exp(τ) + 0.5 * p - 0.5 * exp(τ) * sum(abs2, W) + S = neg_sigmoid.(X * W) + @tullio M[j] := w[n] * X[n, j] * (S[n] + Y[n]) + grad[2:end] .= -exp(τ) .* W .+ M + return grad +end + +function ∇logp_subsample(z; batch_size=10) + (N, p) = size(X) + idx = sample(1:N, batch_size; replace=false) + w = N / batch_size .* ones(batch_size) + return ∇logp(z, X[idx, :], Y[idx], w) +end + +function ∇logp_coreset(z, w) + @assert length(w) == size(Xc, 1) + τ = z[1] + W = z[2:end] + dim = length(z) + grad = Buffer(z) + grad[1] = a - b * exp(τ) + 0.5 * p - 0.5 * exp(τ) * sum(abs2, W) + S = neg_sigmoid.(Xc * W) + @tullio M[j] := w[n] * Xc[n, j] * (S[n] + Yc[n]) + grad[2:dim] = -exp(τ) .* W .+ M + return copy(grad) +end + +# customize gradient for logp_subsample (in each iteration logp evaluation only uses a small batch of the full dataset) +Zygote.refresh() +@adjoint function logp_subsample(z; batch_size=10) + return logp_subsample(z; batch_size=batch_size), + Δ -> (Δ * ∇logp_subsample(z; batch_size=batch_size),) +end + +function logp_joint(z; batch_size=10) + dim = div(length(z), 2) + x, ρ = z[1:dim], z[(dim + 1):end] + return logp_subsample(x; batch_size=10) + logpdf(MvNormal(zeros(dims), I), ρ) +end + +function ∇logp_joint(z; batch_size=10) + dim = div(length(z), 2) + gx = ∇logp_subsample(z[1:dim]; batch_size=batch_size) + gρ = -z[(dim + 1):end] + return vcat(gx, gρ) +end + +@adjoint function logp_joint(z; batch_size=10) + return logp_joint(z; batch_size=batch_size), + Δ -> (Δ * ∇logp_joint(z; batch_size=batch_size),) +end + +#################################################33 +# train sparse Hamiltonian flow (https://arxiv.org/pdf/2203.05723.pdf) +# note: +# - SHF operates on a joint space (target space × momentum space) +# - instead of using the full score in the Hamiltonain flow, we use coreset score +# - instead of using the momentum refreshment as used in the paper (only perform normalization on the mometnum), +# we just stack a general shift and scaling layer after the leapfrog step +###################################################3 + +∇S = CoresetScore(Float64, 20, 400, ∇logp_coreset) +dims = 9 +L = 20 +∇logm(x) = -x +Ls = [ + Scale(ones(2dims)) ∘ Shift(ones(2dims)) ∘ SurrogateLeapFrog(dims, 0.02, L, ∇S, ∇logm) + for i in 1:5 +] +q0 = MvNormal(zeros(Float64, 2dims), I) +flow = Bijectors.transformed(q0, ∘(Ls...)) +# flow = Bijectors.transformed(q0, trans) +flow_untrained = deepcopy(flow) + +sample_per_iter = 5 +cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 +flow_trained, stats, _ = train_flow( + elbo, + flow, + logp_joint, + sample_per_iter; + max_iters=200_00, + optimiser=Optimisers.Adam(1e-3), + callback=cb, + ADbackend=AutoZygote(), + hasconverged=checkconv, +) +losses = map(x -> x.loss, stats) diff --git a/example/Project.toml b/example/Project.toml index a9985022..a1d4a818 100644 --- a/example/Project.toml +++ b/example/Project.toml @@ -2,7 +2,10 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" FunctionChains = "8e6b2b91-af83-483e-ba35-d00930e4cf9b" @@ -13,6 +16,8 @@ NormalizingFlows = "50e4474d-9f12-44b7-af7a-91ab30ff6256" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" SimpleUnPack = "ce78b400-467f-4804-87d8-8f486da07d0a" +Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/example/RealNVP/main.jl b/example/RealNVP/main.jl index 3e94bf06..2eb2bb98 100644 --- a/example/RealNVP/main.jl +++ b/example/RealNVP/main.jl @@ -26,8 +26,7 @@ logp = Base.Fix1(logpdf, p) ###################################### d = 2 hdims = 20 -ls = [AffineCoupling(d, hdims, [1]), AffineCoupling(d, hdims, [2])] -Ls = reduce(vcat, fill(ls, 2)) +Ls = [AffineCoupling(d, hdims, [1]) ∘ AffineCoupling(d, hdims, [2]) for i in 1:2] q0 = MvNormal(zeros(Float32, 2), I) flow = create_flow(Ls, q0) flow_untrained = deepcopy(flow) diff --git a/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv new file mode 100644 index 00000000..7f8dda94 --- /dev/null +++ b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv @@ -0,0 +1,144 @@ +away away bullseye bullseye circle circle dino dino dots dots h_lines h_lines high_lines high_lines slant_down slant_down slant_up slant_up star star v_lines v_lines wide_lines wide_lines x_shape x_shape +x y x y x y x y x y x y x y x y x y x y x y x y x y +32.3311102266 61.411101248 51.2038911373 83.3397766053 55.9930301513 79.2772636977 55.3846 97.1795 51.1479167122 90.8674123313 53.366566866 90.2080300059 57.6132335477 83.9051707998 52.8720214902 97.3432226575 47.6952008891 95.2411869282 58.2136082599 91.881891513 50.4815081703 93.2227013657 65.8155400946 95.5883740582 38.3377571839 92.472719051 +53.4214628807 26.1868803879 58.9744699018 85.499817612 50.0322537946 79.013071199 51.5385 96.0256 50.5171258092 89.1023945866 52.8019793617 90.0880645063 51.2743918205 82.8179829379 59.0141444945 93.5748748733 44.6099759113 93.075835032 58.1960536923 92.2149886482 50.2824056687 97.6099835723 65.6722651618 91.9334018119 35.7518707905 94.1167680276 +63.92020226 30.8321939163 51.8720726696 85.8297376348 51.2884586593 82.4359398425 46.1538 94.4872 50.2074801993 85.4600473951 47.0541298828 90.458936026 50.7538984114 76.7541289533 56.3751090389 96.30514763 43.8563814822 94.0858717713 58.7182307185 90.310532087 50.1867033389 99.6946801425 39.002716451 92.26183816 32.7672179591 88.5182945794 +70.2895057187 82.5336485877 48.1799307865 85.0451167372 51.1705368873 79.1652941091 42.8205 91.4103 50.0694819181 83.0576695262 42.4484337816 89.5077001153 37.0211840116 81.954469997 37.8391996844 94.359443503 41.5789293984 90.3035665387 57.27837287 89.9076067166 50.3269108629 90.0220534916 37.7953028026 93.5324553607 33.7296067755 88.622265789 +34.1188302357 45.7345513203 41.6832004024 84.0179406019 44.3779145287 78.1646280036 40.7692 88.3333 50.5628463386 82.9378217764 42.7040363241 90.4426288607 42.8817550897 80.1847700514 39.8853734176 90.6346620817 49.1774185559 96.6105324538 58.082020494 92.0081450101 50.4562073315 89.9874101286 35.5139005857 89.5991901138 37.238249327 83.7249284144 +67.6707164012 37.110947969 37.8904155015 82.5674929978 45.0102700721 77.8808631153 38.7179 84.8718 50.288527801 82.9752535659 32.3789386326 90.144142631 37.1557104777 84.9541114129 44.0774051326 84.1258533023 42.6522509813 90.5606397307 57.4894477748 88.0852855629 30.4648469308 82.0892320845 39.2194529062 83.5434818364 36.0271982243 82.0407806505 +53.2591294055 97.4757710964 39.5489736947 80.812601774 48.559816719 78.7883706035 35.641 79.8718 25.5834750822 82.9148911324 32.5307027362 70.1573965085 38.7318581261 83.7893564281 31.4970254888 67.9125621568 30.1233318217 81.1442953304 28.088741319 63.5107944263 30.5015964737 82.3071263462 31.5882013862 77.0280524208 39.2392807768 79.2637238398 +63.5149808671 25.1000785788 39.6495738769 82.6645338713 42.1422682922 76.8806345544 33.0769 77.5641 25.4835833896 82.929084979 33.3652644305 70.4593354392 31.0005254108 82.5794805962 25.9525965456 73.0331803541 31.915519986 79.7904134869 28.0854682136 63.5901969481 30.489569227 77.7298801366 32.6577782482 80.1138612983 39.7845249391 82.2605658962 +67.9805388133 80.9571652197 34.7505970472 80.0110909897 41.0269715735 76.4095884613 28.9744 74.4872 25.4435257049 82.8742004973 32.6560991775 70.0510704992 25.9854996226 74.3493875478 36.4347239148 62.2940454639 27.8877193574 75.4455759934 28.087273053 63.1232828116 30.45024833 79.8348944011 31.6415992679 77.6108331874 35.1660293896 84.1564919839 +67.3724659005 29.720400203 27.560835291 72.8478255927 34.5753090115 72.7248411845 26.1538 71.4103 25.5651134237 82.9240977704 22.9550932697 70.4268951523 23.7145662909 75.0980490792 24.997482691 75.4415616626 27.5482344526 75.1647590544 27.5780252176 62.8210386566 30.5183947985 68.2525829388 30.9871426099 63.9440280336 40.622115916 78.5421042089 +15.5607495229 80.0656402858 24.6355399754 71.6107148256 31.1686007236 69.2454213193 23.0769 66.4103 25.9288442731 82.8211841087 27.1498436318 70.2108058428 23.077319507 71.7205396747 32.2462751352 70.1105868383 20.2097781637 67.5166420988 27.7799191139 63.518147517 30.5116612138 66.0937181831 27.4396322054 65.7444629564 39.181907086 79.8190371976 +71.7907676942 71.0654666627 20.9594648125 66.0414983826 32.6442540714 70.7325550334 22.3077 61.7949 27.5514782612 51.4873865306 26.1669092779 70.5003151212 24.9391368744 72.5802858084 18.1094722949 53.7081103324 24.4434528292 54.4736551228 28.5889998149 63.0240805653 30.4980777147 62.3849850338 30.5783699449 65.5219454667 42.4308889944 75.1336346406 +70.2425464362 24.1095975542 20.6891490538 62.7213052064 26.6666456539 62.9809712052 22.3077 57.1795 27.5304663693 51.4148465629 26.3314570795 50.494556525 17.8934987136 71.1096873431 21.7960992263 49.4995295248 24.6308301331 53.3957388998 28.7391414969 62.7208638859 30.6202277519 60.8600821544 28.809401637 59.6210044666 43.0812558617 75.3467016413 +64.9374355444 81.5542049945 19.2882047369 62.0630593582 26.4592623274 62.6346733709 23.3333 52.9487 27.0955703605 52.0767994399 22.0037091415 50.4994401238 23.6730818572 64.8708391385 24.150491069 47.1885335266 21.0379688969 49.829299645 27.0246032407 62.9018588649 30.4921854231 55.8789168923 27.9395185786 50.9850621313 44.3607194344 70.4742070546 +62.2135245453 21.4758389969 20.0245005746 61.3426238695 25.5380076465 60.6302361093 25.8974 51.0256 27.4392496133 51.7120790527 23.2847479782 50.4743528321 23.7490719056 63.557171895 28.6031956015 44.2237227 27.106242932 54.7195531488 28.8013366963 63.389040388 30.474634333 52.3221596388 33.8499497887 46.2844744465 44.3247788435 71.0459044019 +67.2694004772 18.7089683725 35.4695229971 43.1158849517 26.3207111933 62.3962811186 29.4872 51.0256 27.8782642594 50.70890793 27.6340383923 50.5036668574 32.2151786343 65.6009580191 20.9319996761 51.6462354868 28.0896176553 60.512212549 27.186463838 63.5587296478 30.500998079 48.0696996767 32.628463049 54.0637204929 47.552160202 66.4604037217 +40.5701970446 79.3729634752 36.8943276507 47.7065559687 26.1633874648 62.0268148985 32.8205 51.0256 27.3388689181 51.6530467544 32.0371080007 50.5000661229 29.4368412489 67.5545281141 35.4355325251 34.9610381566 31.1357349152 61.4319127116 29.2851466002 63.3836058254 30.4944213623 45.7734755346 32.4386500903 49.9030194916 48.6649246215 64.9610049438 +74.7411813341 21.1016372041 39.0555497751 55.5469737131 25.2137904611 60.2733986018 35.3846 51.4103 27.6784029741 51.1819891691 29.3365106528 50.461893969 40.0500951983 71.8890335708 44.3773828236 57.6847369215 37.700837521 68.4184052804 39.4029453026 51.1508571967 30.4995445502 49.3820997506 35.8382920833 53.3514883396 46.280631958 68.0284231525 +71.7683189223 20.0110618423 46.9570801494 65.2404073935 26.8595295604 63.1418346727 40.2564 51.4103 52.6356576791 51.4185522584 40.816953133 50.4877601957 37.5747950188 69.9887444412 46.313692406 55.6855917703 43.3926884016 43.5975940692 28.8113284396 61.3578540576 50.144392979 47.0142836417 37.1420429198 49.7419879519 50.5632444849 62.6238279659 +76.1669198143 75.9361704048 37.3104527352 45.2587508999 31.7606691716 69.7976553936 44.1026 52.9487 52.0252141055 52.1230110467 42.7390751173 50.483155992 49.3104686263 67.1687603206 46.3860739231 51.0021645381 51.2623974761 49.6665051258 34.3039579069 56.5421259093 49.9128985877 55.5878389121 34.6445467746 54.4441077764 52.6309642318 59.9639357915 +65.6236574431 15.5828033531 40.0096720019 60.9865825077 39.7932905744 75.8079642168 46.6667 54.1026 52.8811647904 50.6215547554 51.8678937691 50.4932732621 47.3480185524 72.5916115408 42.4954400876 56.9036815315 43.2867129203 43.0568603208 29.6027609801 60.157346722 50.3374274859 57.9553142818 35.675404696 49.6256433471 54.6362064446 57.4938424367 +50.8506336394 13.9876016304 48.0143866846 65.7128195934 45.6174391854 78.1199702277 50.0 55.2564 52.9526073133 50.0747390118 43.3710018924 50.5008373234 50.157414434 71.1617258246 44.3005150328 58.1329592781 55.9059162338 54.2526725342 49.1161568603 63.6600006211 50.4189791992 60.5249117346 64.99458547 49.4621728312 51.346027139 61.6005530364 +33.0240700249 24.4678303872 53.7037759268 66.3894820363 53.7714606827 79.3029316435 53.0769 55.641 52.5205524936 51.5024421035 53.0427983006 50.4875038691 58.2100082697 68.0445380719 46.9882833458 55.4206367313 50.198958949 52.2250295948 39.6175458263 62.9251879574 50.2052589714 57.9242171676 66.1291498342 61.6437178853 57.134855354 53.8779658045 +39.7063261674 84.2752871038 63.0674998921 64.0350004581 56.4514336733 79.2471070208 56.6667 56.0256 52.3428220582 51.8619520908 56.6189102967 50.5033740178 56.6089076561 70.4089460888 52.4215231623 47.867005488 55.761587372 54.0305172396 43.23308466 63.1652187223 50.4353151633 56.4393224734 65.790651882 55.9453737171 54.1712415737 58.0598078952 +45.5964849542 9.76334884943 62.0480325076 63.8458632494 66.0937201989 77.0178182606 59.2308 57.9487 51.9275902141 52.2577906065 70.568289819 50.4962361243 52.8338205303 72.5518862725 64.090998189 68.6509850107 58.3231396505 60.0310591646 64.8927879422 65.8141767574 69.5057889042 59.2849248279 67.5017806138 60.8502274659 50.9423823071 62.0973939351 +42.9680469104 17.9454583961 59.8399667124 64.4767644437 56.9258394194 79.2063186226 61.2821 62.1795 52.7137744894 51.1979443188 67.870887292 70.0680261345 61.8185854576 76.1128240228 63.0168717563 68.2252589114 70.171457528 70.2822212385 62.4901493154 74.5842896072 69.5340709833 64.8929773346 67.6295142176 61.4291060701 66.512924456 59.0712818478 +52.4944067819 16.0511142003 55.1609418175 65.2373081694 58.9882538467 78.9172589554 61.5385 66.4103 50.4438027888 82.9418288173 58.8215465891 70.4557817483 61.7276334143 77.0692384058 67.3025779226 64.4134627173 60.600276471 59.7533088802 68.9880844294 63.232147305 69.5101069302 61.3296244282 63.9100640599 62.9723349305 64.3979095023 56.1748754008 +46.0822757831 23.1104578154 61.2797865792 65.766402504 57.9022660533 79.0808678187 61.7949 69.1026 50.2166950326 83.7523429676 60.1869278445 70.4835512458 62.1922520815 76.1568192103 67.2259568904 62.4764105301 63.7602750188 61.1856411696 62.1056186306 75.9908707599 69.5263228876 56.9952791254 66.5493579687 67.7255886599 68.686593294 62.2373439902 +74.2477082092 20.314187812 60.8349175348 64.6037697063 64.8128669568 77.4869389235 57.4359 55.2564 52.1841801076 51.9752506714 59.4355174849 50.4206326073 58.1187658587 73.1189614684 50.309988715 47.2678818553 54.3894952619 53.811901225 32.4618467439 62.8819029189 49.9472399619 48.2295767261 64.0515417112 48.8200939835 65.0277937057 57.0962573525 +64.5682641863 83.6396338956 61.5205906458 64.7918550418 60.3497935166 78.7294458673 54.8718 49.8718 52.7920973459 51.0733956509 49.5858434795 50.4349813825 58.2704123843 35.5968950425 45.4126389249 53.8707976895 55.2412969947 52.5935746267 41.3272006535 49.0702512739 50.0631041291 58.085037556 68.1932895833 46.5435408096 53.3045031712 40.883844549 +74.0216939058 76.1282745076 36.916543863 41.0952474383 48.890555756 16.8065435154 52.5641 46.0256 52.5897198565 51.338090196 41.7797417848 50.4706422808 59.7665303129 32.6212516009 54.0206071815 47.2698972057 49.0701267502 48.8969446811 44.0071499323 46.4496737846 50.489902373 39.2054119682 65.2783891952 44.8203529292 52.9451780881 40.3846256942 +62.3911805626 5.62307076073 38.5021996714 41.5671556174 47.549506348 17.0714045245 48.2051 38.3333 52.0288486671 52.1768375002 46.1276863394 30.4494107792 52.5355347207 27.1741229419 49.8420222483 50.5765522936 46.0181190216 46.2577233718 44.0740606888 34.5532038906 49.6888638379 42.018514414 34.9976613883 28.6329939351 45.225738199 30.5552259226 +74.189036683 68.1335832223 48.6643707344 30.6806668519 30.8894308257 26.3290372348 49.4872 42.1795 52.729246581 51.2017650475 38.0337837705 50.4908995189 50.8925215809 29.1452839382 50.7132995657 48.7722867825 44.7595874356 41.9746066134 44.0013152386 33.9042073544 50.3334811097 43.588671479 33.9237661692 53.5586885227 50.8433046202 37.7446280981 +28.2367819396 56.1395964513 50.2852524042 30.3379221051 28.972835724 29.0504689185 51.0256 44.1026 52.8843132864 50.4414354501 44.5468412844 50.4159133758 49.9675511775 28.4602033343 55.0455549106 45.9862727459 51.6006241332 50.0324668853 45.0063004454 38.2990195507 50.4467514196 57.2305889423 66.0044543688 52.6846691452 49.6592939398 36.273442735 +75.7719387944 69.8292300322 42.276332666 34.5276361236 35.1613607523 22.7868496473 45.3846 36.4103 52.5093008862 51.4162051508 46.2381175975 30.4956129408 51.3478832298 27.3642602259 37.5437012597 32.8840244657 36.4658820182 35.5241876075 44.4438406096 36.0190833012 50.4209091963 43.3479568967 37.0232562464 32.1184816794 47.1427889348 32.0250641734 +75.8552294691 62.5170442862 54.0317756204 29.6723483149 45.8703691685 17.4893933655 42.8205 32.5641 50.8626843321 17.1456310914 49.7397453317 30.4727248868 37.5551162883 24.5376638386 39.7958978066 28.4432191156 33.3947987327 35.0168013145 42.1787133986 26.492119482 50.3740520508 32.9182406167 34.6982700852 36.4753541349 41.2700565259 21.7218171465 +65.9708570175 72.7448559954 37.3293552552 39.6020423074 32.8314206076 24.7975381459 38.7179 31.4103 50.8914922458 17.1413237277 39.1102518936 30.4786951789 39.1995941778 23.6804956588 28.5233910215 39.7966662481 37.5800715749 38.8315722007 44.0445656189 35.6622382764 30.4768299051 39.4246055913 34.8634441715 30.9392745218 47.1301271555 31.3099029222 +21.7780404779 6.61662530728 41.389522551 37.2960562328 39.4855747899 20.0330500489 35.1282 30.2564 25.8551275976 17.0819086886 39.1428377913 30.7928559394 36.3827221955 25.2380052325 31.3996460012 39.2652632229 36.619722059 36.8114141094 41.6404540171 27.0930954213 30.5034743497 38.3485561381 31.0900650789 32.3837547196 40.9159715783 23.5415923929 +67.7597962473 72.4212015285 40.0746666572 34.6236852027 33.4461829314 24.1906434075 32.5641 32.1795 26.0256445518 16.9250135319 34.6254702671 30.4997410297 41.1006943822 26.5526132161 39.2415652162 33.3614166846 36.1634664356 33.819314199 41.9383300069 24.9915229793 30.5048349283 28.7714879939 34.9213061465 35.4184380529 38.6601639737 21.6895496369 +78.6171953363 52.5752573142 35.3496806211 47.1410731332 21.8635812756 46.0525933457 30.0 36.7949 27.8931727187 50.6619634101 34.7542074707 30.4958650002 28.2178118248 28.2802924847 27.7140565712 42.2500516253 33.1677570349 31.2477411672 44.0539275103 33.5563924949 30.4622184647 32.7504720275 30.343361183 32.5245631742 42.6125508262 24.195053774 +68.5077081898 15.4569189652 34.7637004189 47.6247999245 25.1658708449 35.7666139539 33.5897 41.4103 27.6399679354 51.3990974838 36.7537002386 50.5293043035 33.7970720433 28.6997507433 32.9607035213 35.8231112903 42.6449829318 40.1543216386 39.2067193308 51.5337156971 30.491302622 43.5613740753 35.4833288685 44.4916349303 41.2090395008 24.1287091891 +74.8850211598 25.4166063231 37.0266294472 44.4622930512 22.9594789762 51.2877067934 36.6667 45.641 27.892658895 50.7952815163 37.0193512929 50.4926666972 43.9193873055 27.7978445882 35.4047362397 31.9391621952 47.2250252695 44.5756466185 28.7044492315 61.7775254006 30.449653838 44.9951230008 33.2349104271 38.502483721 50.6492211499 37.4837907435 +66.4549036599 19.8366286542 36.4555695327 40.791843032 25.5860552415 60.8465833585 38.2051 49.1026 52.7977329399 50.6860370901 41.4171152753 50.4697734181 44.4164011084 28.9476923345 51.6422264225 50.8383406788 48.3923630032 50.3315285645 31.7086628996 58.837754374 30.4859942157 47.3479464653 35.4399436182 47.7975176939 52.1319019406 39.3646020815 +77.3178020985 48.3983464352 35.5376642131 48.7293868661 26.3143100672 33.2971681038 29.7436 36.0256 27.5806388053 51.5247612607 22.8171074784 30.4360403387 28.0376027706 26.8038243613 34.207620843 33.5938054385 27.2451932001 29.8195074488 42.8117114739 30.0204484197 30.5020767935 36.4412129198 33.8735095227 37.056028453 39.7972712705 16.083549649 +58.9124603193 75.6677562173 20.4089478892 32.2030304205 33.037301251 24.5644624921 29.7436 32.1795 26.4913985349 17.4053909738 34.800701463 30.4979313851 31.6895761296 26.6664002089 36.5234892198 35.6710483891 30.4043143388 29.0795399241 43.300614891 31.5264261979 30.4780588886 28.020580489 28.5641377966 34.6263997171 43.5498814807 24.3072109395 +57.617447817 8.19480060319 23.4957104659 25.3224681542 36.4688894772 21.8761144725 30.0 29.1026 25.9853178159 17.203722132 27.8918897953 30.4849702595 27.2796128523 19.2572809685 28.4192021281 21.3157482481 30.0807992865 27.5880255173 40.3986329069 16.3470083822 30.5011608787 38.3141385582 31.3195486706 31.6639905316 36.7146633371 17.3818098572 +76.0882257967 59.6799300235 29.5575433634 21.3647774591 26.5135531009 32.9187498178 32.0513 26.7949 26.2014192753 17.0938239128 32.8706391587 30.5023417488 31.2218783581 26.4408557812 26.1604969417 23.0422314726 33.9806339675 35.0330581489 40.4356915763 20.2326706762 30.5047730174 26.4166256707 30.796174379 23.5190343602 37.2246101278 17.9800644828 +57.4660505497 1.50441817488 33.0082339121 15.9850714584 36.0270912125 22.1954385395 35.8974 25.2564 25.8575694746 17.113842659 37.7940494081 30.4926383364 33.5458051034 20.0807081014 40.435327397 27.6400711421 34.2506117649 34.4415865654 40.9365466658 16.9130048364 30.509115326 22.8255471196 34.2333554347 22.4959116015 40.5124478537 22.9622145725 +79.4283834934 45.2107942872 53.9803991821 29.3509867092 42.6810404129 18.5425361977 41.0256 25.2564 50.7046843629 17.0237445436 35.6673163042 30.522035054 39.7363306065 23.4948069772 44.544955314 27.3622426221 51.1411835714 20.6183237784 39.6615736653 15.609355577 50.1156321728 16.6940065479 36.6307877065 31.4768312621 40.2220627884 20.6146217044 +76.3565221496 10.4182411281 52.2343086004 29.7116729865 45.0410771273 17.7345177303 44.1026 25.641 50.8119753465 17.1149252621 48.4422995026 30.5017227753 42.8418426525 20.3339004396 40.1237272008 30.0731619402 49.899592066 19.5640835356 40.899259175 20.7985289466 50.4483013392 21.6402564599 34.5486191018 26.1110878654 44.1249612057 23.3662452858 +64.4050752632 78.5841760758 59.5030766063 30.6696739448 47.3524164557 17.1347454264 47.1795 28.718 50.5648455577 17.0777773216 58.6693732631 30.5068378948 47.0937298879 26.9934706312 42.1848064503 28.5736747331 55.5898773424 26.2992451143 41.9686168283 26.4970725985 50.17317743 29.177864393 37.5196356 29.2676058936 42.3462616039 24.5583269437 +40.6350418091 73.3947503698 41.1637810689 34.3157582514 53.8461940322 16.3873180961 49.4872 31.4103 50.9393039127 16.981021883 47.9169554375 30.4867257545 51.4536583539 26.0229417521 40.0026592892 27.4875100027 55.4820563581 22.8274662445 40.3834058238 21.3912255247 49.9227419194 38.4404729124 62.4256973267 34.0442062707 44.1890843708 28.772625473 +43.9498645857 75.9587156671 48.9930401201 32.0303588363 55.1296393835 16.3532468711 51.5385 34.8718 50.4588548392 17.0385789686 51.8941663939 30.494755684 49.3677068167 22.005580703 57.1252310509 42.7390191388 63.5729393821 32.9386176053 56.5381264538 32.4442454697 50.3112993746 26.8604882737 64.320383794 31.7856294846 45.8142453101 31.056979672 +30.9962205791 71.694404938 59.2692803248 29.6407017654 52.1881199617 16.4182174652 53.5897 37.5641 52.9013640729 50.6905627231 51.7490891643 30.4729933399 48.2534809736 29.2708555331 60.375744968 41.3564579961 59.0250770848 28.3220858801 52.9706912828 29.0401966941 50.3994096114 38.9638389234 65.4455325117 43.2734991936 46.6431192691 31.4072190618 +68.2307689907 80.8725016628 45.4691770035 33.8311927323 58.6726630845 16.6633724322 55.1282 40.641 52.6849534438 51.2944692191 58.2465530013 50.4428744949 61.0029597643 30.0153144324 55.1058961252 46.1490510561 62.4237785929 33.4865952868 54.6209525885 30.3445244537 50.3373626448 40.913386144 64.3256315887 41.6159763773 52.1898883047 39.3677964938 +72.0463894612 12.9180067349 62.6912665406 30.2903738312 82.1443116037 33.1952218669 56.6667 42.1795 52.500088943 51.594356165 57.3106968674 50.4825132895 60.8065608049 27.8683134861 61.7372710245 38.5848440052 47.849907703 48.556624645 65.0990443941 27.2415575588 50.2023180965 47.2129480045 65.264338195 35.7175447772 51.1003173587 38.095699984 +46.5927679682 84.9723827774 73.4286708654 48.5778551303 75.6865711383 24.7415103908 59.2308 44.4872 51.8356372562 52.3357655339 67.9680481127 50.4521589097 59.0674270249 26.7882840217 55.7412443246 46.6535769763 52.2685616825 52.8052401961 63.0559909076 29.7090956748 50.2749326091 49.6234993439 66.388704175 38.8447443366 65.4008341477 42.6140320392 +49.2572183396 81.8814032306 70.846426107 52.282253331 85.344964503 52.886419866 62.3077 46.0256 76.9954121013 52.0455286534 65.9493192681 50.494240794 61.4146337175 31.0393750583 58.2964084158 41.5831977346 68.8510215758 39.573434273 70.9601362323 41.2595012867 69.4943803061 52.6124107059 68.1914228192 45.1694864796 64.8520398267 43.1337348404 +42.7817612539 12.9911884302 71.5390198495 45.5218061588 83.8446535127 58.6160602305 64.8718 46.7949 77.310600475 51.7467300374 55.7470928102 50.4789748003 68.9343694692 31.6198930866 57.0425482213 42.418757739 65.6763960251 35.1093154301 69.8958192404 43.4537592655 69.5193397556 47.384533707 68.5406624515 48.4385263389 63.4038060118 44.847493688 +65.475952195 14.2745856444 67.6208658884 38.0065584658 85.66476066 45.5427527692 67.9487 47.9487 77.9260443352 50.3186604237 68.4030100436 50.4986272318 63.0652438592 34.2357465852 76.7756112073 55.2545987009 77.7468851148 47.3718241139 70.5958928563 41.9647438672 69.5291826079 57.8412666804 67.6614312657 51.0685845873 65.3715488709 42.6959713164 +71.9650826544 17.7102359443 72.4709525633 51.1221348241 77.931004934 68.6900618808 70.5128 53.718 77.2543883433 51.4618248152 76.696465628 50.4938305609 72.2725228706 67.6444437851 71.8870772602 59.2669926152 80.352934115 47.8002988852 69.6470214273 44.044445022 69.495755496 58.785073162 68.6755160534 50.465234811 61.9080397046 53.1903520694 +32.1464623358 43.4817094425 64.8122375621 62.8109155873 73.0318330209 73.12057473 71.5385 60.641 76.2431578028 52.1236898503 67.2148371159 70.2600284356 75.3598819664 75.4216145799 73.7883730663 58.6208121164 60.3375864858 63.0349593272 77.392982494 63.3714590552 69.199057706 60.4189573202 69.0817099788 55.4740530987 71.8266529973 65.6862303604 +31.8384976954 71.8121653901 60.8536798727 65.4991470327 70.1028044999 75.6832068542 71.5385 64.4872 77.0844843669 51.9671367003 72.3020057291 70.2204373812 72.8336258709 72.1146938095 75.1396847343 54.2445521781 61.6587233282 64.1000234779 64.4007871926 67.4487184472 69.5510914402 58.4010982686 69.048716493 66.3660457477 72.6546231621 66.8782776113 +31.0052582572 40.682503007 67.7894961571 61.3637015467 57.1446946889 79.1941025922 69.4872 69.4872 75.2280531954 82.9856620201 66.7632573944 70.4738617113 70.8104148392 79.3504618327 64.5963189058 65.952512925 71.5450389858 72.657716331 63.8689598271 70.2137388333 69.5066843151 68.0844667893 67.3082233775 71.5902965807 76.8885781632 72.1654720777 +80.4708943189 49.5021483467 41.6095572691 83.8486865559 46.9624619942 78.6665589963 46.9231 79.8718 50.6583547714 83.1144793358 51.6803004439 89.9970275485 45.8019347617 81.6582677456 46.9095386323 85.6022611586 49.0409566406 77.3625832678 56.5944213157 86.9270062202 50.4441175504 75.0132995059 38.714899408 76.3969778172 43.7446196844 71.2679610751 +71.9641671122 41.8742826668 53.0030253202 84.6747986012 50.5649338128 79.0772985529 48.2051 84.1026 50.2033658114 82.9826568621 53.0965573571 89.540243712 45.835020171 82.7425919919 46.9051031994 81.5656424136 51.8868880222 80.1136076892 56.5313372853 87.4998110713 50.4672293846 90.281464505 35.5480948795 89.3049758407 39.0542484428 83.1190207043 +78.0794214417 93.1333167652 54.7141710591 84.0431280734 53.7196765696 79.306936167 50.0 85.2564 50.9295476993 82.8460411252 42.2422929389 89.7085785936 40.0788064496 80.2471852579 49.8069877356 82.085025592 57.968981435 85.3560457326 59.6521583697 87.8094615921 50.210881979 80.9417717084 36.7106729424 86.0749670003 39.9316746322 79.9391880033 +41.6775957748 30.2012640846 44.2916687157 82.9094412144 54.2002323226 79.3250023354 53.0769 85.2564 50.1786718465 83.1846223323 56.2338756577 89.635308157 51.9829247003 82.2043834675 48.8632418189 83.1583574877 54.855664646 83.7833976656 56.6365087005 85.637495556 50.205406955 90.6405359009 64.9126624396 87.6457960566 39.762826107 76.4059674459 +65.953595185 31.1474060835 49.1917219555 85.8762291224 47.875198343 78.6758534295 55.3846 86.0256 50.422698058 82.9099014671 54.6809865825 89.9645994589 57.337906053 84.9995188696 66.7481899024 84.0048341677 54.1331954613 83.7939960824 58.6722880025 90.077160307 50.4382906648 92.3547602661 66.0641357377 89.2086685815 41.323383287 80.2118685969 +62.9344593731 31.9163906992 53.1013817819 84.5476586881 51.4734585149 79.1598669963 56.6667 86.0256 50.4642248341 82.9353284131 57.9026866088 89.6184017294 62.3362327624 83.2465678988 62.3597662841 90.2436599196 51.3088252568 80.1363911553 58.2216127264 90.4110187715 50.3735188462 82.2192057218 64.9359294757 87.0721381174 41.7873232275 83.9880779935 +64.3737979844 28.8625834061 51.599848153 84.8198214898 52.7512222162 79.2936599344 56.1538 82.9487 50.4492703306 83.9699203811 51.8202625467 89.8341303987 58.5059100905 81.7455863409 47.3748964954 83.4715555182 61.1203943844 88.7040529069 57.9146644768 89.9538027677 50.253683689 81.7438287242 67.0265507063 76.2225153324 39.7556490651 79.6570966361 +72.5093283599 39.5401302526 54.3797219484 84.2403555494 48.5622230047 78.7823835275 53.8462 80.641 49.9283802752 82.9936654888 56.8283953362 89.5836647251 56.2479351377 83.513949493 47.4464667165 82.622124014 55.5638650304 85.8103751441 55.3155090581 80.2518606944 50.2504708592 71.9513852104 65.3140322257 77.9387440331 40.7498793985 77.204657286 +30.0522898741 96.6175423534 46.4807681047 83.518211666 53.0812416908 79.2972507654 51.2821 78.718 50.4880136436 83.0995191232 58.2722785867 69.9002729578 55.4065208295 80.2666719669 53.385280634 78.8711362558 55.044123361 83.5414106324 54.5757285877 77.5362884724 50.0870583609 73.9518736268 64.2952177764 84.1258572133 43.8568984983 77.6098436482 +28.0033242354 46.6721919544 53.174656268 84.2605679876 56.475383374 79.1678400979 50.0 78.718 49.9649053793 83.7083177013 51.8281274487 70.3526492053 53.2839688423 84.2635992213 49.4527077733 79.9306028004 50.4441853247 78.1958883115 54.4130936504 78.2290865878 50.2310718829 80.6473870602 38.7980121423 74.6106304299 42.3678224956 73.6382718294 +75.4012268619 88.6390766207 45.272002941 85.2370794195 49.570911405 79.0381238831 47.9487 77.5641 50.7521082559 82.9019500961 44.8499842618 70.3952029719 44.6767615455 80.2382855483 54.8026825363 75.3639849893 43.8277089981 71.0470522123 55.0745059041 79.8175464166 50.2077411201 73.1145035177 34.6757573144 79.6375987743 39.4629598364 78.459973898 +38.9800154218 87.322160691 36.03340215 53.371687755 37.6277032991 74.5575043418 29.7436 59.8718 27.4224292143 51.4388762325 24.2703823928 50.4855676918 30.0112642472 68.1467563743 37.0882451339 60.857227495 37.7914323802 66.8546410275 29.4329605156 60.8017765444 30.4995061735 53.6125276703 30.7192619427 58.812060082 44.5173339492 70.3693498252 +65.2199135479 84.6829549336 28.2711941653 72.840231257 32.2637000863 70.3166389012 29.7436 62.1795 27.6740833991 51.3041121452 34.7424714403 70.3399348879 29.7452539206 74.4045268347 32.3759619238 68.5108204552 29.292783111 59.79199693 29.4226860665 63.0684648229 30.5203612407 65.1004090686 34.1051509268 64.1291972267 41.6360929963 76.2492229205 +73.0539899616 29.3808085571 25.0548060847 71.5485979221 32.542380336 70.7024172063 31.2821 62.5641 27.5373953151 51.5936540762 37.7993149267 70.4695358763 37.3099483549 72.5861552003 37.4770138847 63.7886855744 32.1920969515 60.2006509147 29.0056141576 63.3907513334 30.4549558873 63.5017787814 33.4233718511 67.0957433613 45.4229321817 69.0955016709 +34.3983616372 59.5444469033 64.7588870014 82.3152236405 52.8250171923 85.5781338795 57.9487 99.4872 52.2633473764 94.2493278274 49.7869410649 89.6580469945 61.1359469093 82.5375693646 52.4066488967 96.4254150057 51.4339355233 99.2568672885 58.4618385916 90.2653263904 50.4164772292 97.2696919618 66.3138096741 98.9310270371 42.0028562742 97.8376147235 +43.4904501336 40.782542065 63.1445274767 85.2366963264 58.5282895407 78.9787042541 61.7949 99.1026 51.7372816586 92.9791175339 59.751497414 89.7319325216 61.7206191907 85.7056934492 50.2970432655 99.6441791736 67.4641398118 95.999963905 57.997804739 92.1599086113 69.5250693831 93.5980502089 65.3096314498 98.1651568396 76.1056290282 95.3049805754 +55.138737967 30.7257603575 50.4246786898 85.1747475891 56.265834263 79.2529394854 64.8718 97.5641 75.8709636929 88.3864417377 64.7747992444 90.132316736 63.6093798555 83.3600288362 61.5633243835 89.8641388605 65.1507052757 92.5026337368 57.5494740761 90.7489065581 69.5202566002 97.1938760167 65.7140760682 99.2837639519 79.2025626772 92.4072156264 +43.6843934333 32.8230098696 70.6449962629 82.4309187567 57.0758924193 79.1939225382 68.4615 94.1026 75.2443262052 83.9034973788 70.8444476004 89.9242090159 70.7237162322 87.1522075165 61.0865603919 89.2243556475 65.5834276962 93.5170609365 59.5299284566 88.3272741451 69.4989700668 92.1204899292 67.5934531877 90.9519098246 84.8482440854 95.4248045304 +35.9036097344 91.1118630801 63.1490490834 83.9468553485 54.2564023808 79.3138648504 70.7692 91.0256 75.1982952889 83.4623033365 70.6059455116 90.042061963 75.0063926418 82.4258351563 59.2364904011 91.4811005766 69.2889731377 99.5795911254 58.2493910631 92.1296814778 69.5286156637 91.6220882409 66.3880173202 89.2812304744 81.5644753207 83.7929072262 +45.3780188805 29.1692166544 62.8240245172 84.9661859507 62.6856712475 78.1747432719 72.0513 86.4103 75.7010415321 82.9194588559 78.6424883907 89.5512032307 75.0925497814 83.4373907092 66.1540050863 84.1119370252 81.2694405511 80.635944933 58.0245140126 91.6944211685 69.4997669273 84.47968329 67.8061881135 91.6881773501 85.4461864009 83.0782938448 +39.7774828713 43.75581895 70.236869515 82.1711510621 68.0856058616 76.1123175001 73.8462 83.3333 75.4793396586 82.8840513891 70.810692254 89.5926540118 70.3279291834 82.0293546968 67.2313347767 85.055640808 78.5757448361 76.8157030942 58.3821244904 90.5534760692 69.5038013697 81.8154056166 70.9626103129 86.5181213517 80.3124913784 76.5162389518 +38.6644611569 33.3172384774 70.0427352449 80.3850213547 65.1181219632 77.3991884543 75.1282 79.1026 75.1945668739 82.9321157846 66.7699466213 70.4233752108 70.8633308324 79.4081747457 73.7099068702 78.2706108976 74.1867734997 74.3442041467 62.5667590405 77.7439347591 69.4875893963 81.8359875567 71.9936371714 78.8209480695 80.7662322969 79.8960656796 +39.0440366877 84.6760108316 72.5706234458 80.9712184346 68.7456312971 75.7900554548 76.6667 75.2564 74.8202539551 82.9623887856 73.3363681612 70.3169271705 75.7799048195 79.8861994199 76.5804513576 72.8915336827 72.622209414 74.5269656511 72.1758243064 63.1289294176 69.5191157281 71.1159672213 72.4798343312 80.8006830564 82.6701124763 81.711479845 +91.6399614428 79.4066030605 75.1307160373 79.9840931403 63.0902251412 78.041409495 77.6923 71.4103 75.1643404933 83.0349971693 73.1051481627 70.300511881 78.0936590733 77.759558184 78.8765811639 71.5625276804 75.1415195918 73.8196684648 79.4727615693 63.4086861199 69.5043762947 70.4934438205 74.0784292286 72.9515308522 77.1636254828 73.5237189623 +47.4881326771 85.3899333808 83.2939057345 70.7784317876 77.2045893204 69.3121497658 79.7436 66.7949 75.2633555474 82.9452793018 74.6782541432 70.4453141903 76.7457612243 71.9733600919 77.4490021363 72.4473094857 75.168667839 73.9004448762 80.3577008812 63.2954375404 69.5125730109 61.497563947 72.0362880189 69.7160484161 74.0707000309 68.4795642895 +44.5902125769 22.0340116412 79.6642622775 73.9323097157 82.0339084758 62.6818697859 81.7949 60.2564 77.7564189331 51.1517703255 80.240250951 70.3543089613 78.4646034437 69.7493008209 86.8537073476 65.5433240194 85.3242028911 55.3885788252 78.7572361375 53.3326200111 89.4942005622 62.2343269676 72.9981862717 59.6656447092 74.1142308687 71.9263635185 +39.0896145478 70.4661940802 88.4321025254 64.6242470023 81.5779459183 63.8949255005 83.3333 55.2564 77.9544350549 50.4755789715 70.9468490729 50.4727017878 76.7456848486 70.0440572588 79.806531301 52.7231797883 84.090507598 56.301968006 82.5402395934 56.5410522935 89.5001240511 53.2163306165 74.8121312339 57.5898837631 68.942075458 62.635150577 +42.2293783752 19.9140684075 89.1155590082 64.0015066388 85.1801432805 53.7816766607 85.1282 51.4103 77.0833377719 52.1577992662 88.6626638086 50.4553260208 85.161682187 65.5124047088 76.9882734745 54.2027848022 86.2562238205 55.8309303793 86.4358971909 59.7927618087 89.490246254 48.890796551 73.0273050243 46.2741878725 73.2420023309 67.3532711288 +37.0003871448 60.264279248 89.0921976305 57.768193047 84.2868249936 57.2561680896 86.4103 47.5641 76.0635502513 52.1046520613 84.405952832 50.4948658866 91.8083525691 34.9655944844 79.4793575017 49.1410154455 92.5487933153 42.9019079116 79.4886844186 53.6516742641 89.49778562 46.6299066746 77.067111934 51.4869181896 71.8670131631 34.3291575887 +39.0520864793 70.6525028457 91.7260057727 52.6233532573 85.6224867126 45.0241658646 87.9487 46.0256 77.6820163247 51.1656378096 75.9871047068 50.4919669557 91.8874485924 34.7692538555 82.3041595613 46.9909036215 90.3426345089 37.7803861004 81.5304203188 56.0253645659 89.5048511306 48.4234081578 75.0171368727 46.6224426489 71.5691888532 34.699218607 +37.4884147432 60.8144048511 91.7355387598 48.9702108877 85.6044990156 50.7623379055 89.4872 42.5641 76.8780819814 51.8675622958 91.1120854191 50.489389393 92.2484016622 32.3771544295 90.700691247 41.4667030063 85.8191629042 37.5713861819 79.1867885665 53.2347918468 89.5010526463 43.3473436951 76.6653077326 38.440250966 71.4829096934 34.6805169926 +69.3595594592 65.5213545959 91.5078881747 53.3126520851 84.4247858276 38.761410164 93.3333 39.8718 76.9485027211 51.9075165439 98.2881232748 30.6039193719 91.782532741 34.3842946521 95.4434878077 36.1897016148 90.4418181137 40.1331134204 77.8990579454 51.8224583343 89.4888412549 40.6250912209 77.9158742329 45.9268433826 78.1285220512 23.139231835 +43.542775926 62.4603112824 88.239001904 31.4774348786 85.570985078 44.5600964397 95.3846 36.7949 77.8640547124 49.6625455341 95.0652748396 30.5004028652 96.0805193672 28.0536005018 94.9974880471 35.0953011066 92.0175590822 39.9179311324 75.1307142075 23.3724419733 89.5016235276 45.8151791904 73.7420460836 39.1209853231 78.789825254 25.5971933466 +39.8112302539 65.3348328092 88.5305192041 30.4760310098 84.6267242006 39.4870775084 98.2051 33.718 75.7714500936 17.1112512057 95.2492339568 30.4594542144 92.2478988278 28.0707514904 91.517847246 36.8902631263 95.2605278366 46.008830272 76.0580137544 16.3837496882 89.497529662 33.901372671 75.3298157891 32.8303519164 77.7815467962 23.4472144745 +70.0689259404 7.59346560899 55.3651603364 30.4458502777 73.3961549252 22.8162479426 56.6667 40.641 52.3315691301 51.8788603463 49.6561262881 50.4313198677 57.662276619 24.0606079328 62.2926421901 40.0295333141 41.2902998033 42.3115524113 57.6146743876 33.8224476502 50.4390920216 37.670771798 63.4104355141 38.377735614 48.5306293681 34.1524500077 +70.0405435824 77.1438066024 62.5602588694 30.4471356661 73.5074477338 22.8569887644 59.2308 38.3333 52.5928183721 51.3915915185 63.0186009279 30.4925819262 59.8469248931 26.1777124879 59.2034756073 41.2706776667 66.4313243161 36.5361097353 56.1713975295 32.1179887719 50.179850128 36.8075593352 68.856486669 43.084147214 65.5307814724 42.2810847015 +39.505789079 74.8516272173 58.0066691227 30.2537212987 63.3564881135 17.7240688424 60.7692 33.718 50.4770477213 17.0482894119 70.5382006004 30.4835582526 64.0670962137 27.7765005568 65.2563927704 32.8541041237 63.9863468858 32.7134316129 66.2878905956 26.1171097453 69.5050706711 33.7401525035 66.3377850519 33.3065100022 70.1180341426 36.4481847905 +62.5168908529 66.4847322418 55.0671179917 29.0115351997 72.9590468722 22.5114998325 63.0769 29.1026 75.2964750909 17.0156531888 70.0868697658 30.5100448173 60.9815013975 23.9342981582 62.4447424979 35.4149992261 61.2696820905 29.3857240482 67.8817196169 24.2360184109 69.5798670452 21.9352367455 64.2037185014 26.6441143003 70.4722626065 36.0880530553 +72.1399254065 0.0151193251552 61.6147759612 29.9943911942 64.0208416469 17.8835812521 64.1026 25.2564 75.573954129 17.0621921415 62.6856904424 30.5464245575 59.9670579092 26.8025832185 72.4624062907 27.6242638254 60.7493891909 28.0517363383 64.0280813016 27.6726855125 69.5064719492 22.7286628748 64.4986348346 22.863501327 76.7911300701 28.0720081685 +45.2515760666 70.0034213192 68.5431435378 35.6578321706 61.124463241 17.1163533592 64.359 24.1026 75.4005271583 17.0411068868 69.239417959 30.5060052447 62.7074541119 21.9955085787 69.9194262552 29.3110927191 64.6552305634 16.6122765086 77.4966517463 14.9485235577 69.4994788728 16.3374699821 68.8909906096 27.2962056672 72.4093545935 33.0392569664 +42.0633045627 2.33519661206 77.7061096487 20.3042601852 75.6742073297 24.7825009657 74.359 22.9487 75.8709908356 17.1348939075 80.4002336791 30.4850063746 69.1952352018 22.709792166 77.7628590782 23.4913735562 68.6382807138 18.5080128963 77.6346517616 14.461853927 69.500264751 27.1360239942 72.371523338 21.9616397473 76.271299815 25.5695441452 +36.3556951539 6.0058486497 68.4530459999 13.0355285908 69.5135831574 20.2793578927 71.2821 22.9487 75.6058847579 17.0677230553 74.6478782284 30.6738628584 65.5923626471 20.6941731019 72.7784101997 26.9005353695 65.7766651568 16.7876560655 77.8637264289 14.6106776491 69.5041565157 23.6689181006 69.7654249704 19.999850454 76.9532591384 27.7706107405 +30.3918276596 42.75961287 68.2572064402 12.3846391542 58.5721354669 16.642703808 67.9487 22.1795 75.8955770466 17.1699497051 65.9952727194 30.4750241915 64.6170985343 22.4431455531 65.198316667 33.0271717859 71.1341263421 19.3859575313 77.3381581659 15.890054658 69.5322795847 21.8509008151 68.6213124429 18.9156764428 75.1077300904 30.0169834157 +36.4490038543 50.5462690659 70.2554746739 13.2503849689 70.9819409865 21.1458881039 65.8974 20.2564 75.7465632023 17.1057165115 67.0885590036 30.4945239508 59.191159132 17.767901807 69.4598588916 30.1970315053 63.7231835039 14.5214211957 76.1804165329 15.9125737458 69.5001114942 11.7916260054 64.2977371716 20.4287496884 75.9692852004 29.0388653753 +40.467576002 60.0275120878 65.0443252778 11.0008414785 65.2230303973 18.3971951223 63.0769 19.1026 75.1423414811 16.7549238932 59.8525770283 10.5431877763 55.8396313724 21.6185696663 59.3150555205 12.3543136666 73.0014940701 24.0738125371 77.2526510945 15.1515170246 69.5048154243 15.6143582625 66.6992650902 18.5910853404 75.2838870131 24.8747531721 +81.7246168002 6.03754484635 60.0922466078 11.8721117093 65.3150439093 18.3337482986 61.2821 19.1026 50.6617795563 17.0781405153 53.7489900744 10.5225745323 59.0007586238 21.1710074885 51.487713355 20.8436106875 72.436657497 20.5783451136 77.4133752817 15.2219279762 69.5034745234 17.1690897689 67.5445276811 16.4479381344 77.0522406385 27.9056390894 +48.8231974964 76.6353305783 52.9920289667 9.90666848335 64.0102406778 17.9244781912 58.7179 18.3333 50.6998506392 17.0851843836 58.2390955164 10.4948990284 56.4593711284 25.1445642082 57.1124475509 15.3217851726 68.4127671225 19.9131265443 76.7318493982 16.2168461441 50.3774019862 16.5060150613 63.9469519836 18.6928454476 42.8267563374 25.5569594065 +35.6205617651 57.2860155789 50.1446289773 12.2115430946 57.1409359294 16.5986105437 55.1282 18.3333 50.9189408654 17.1476047571 54.810300495 10.7249292845 52.3581053558 19.4170541222 50.2244178157 19.1709698793 52.8583150204 20.2350610035 49.4711054118 25.0630193062 50.3394254813 18.7234506861 64.3881920705 15.7728122808 42.0806606852 24.9465264454 +50.5839631148 71.8066161014 46.5086141932 11.2071344881 56.3651532552 16.3265463742 52.3077 18.3333 50.7252585405 16.9074698083 52.9767123211 10.859490276 51.8116637782 17.0334945954 50.4338611907 18.5348708515 53.5299526734 24.2419155845 42.4765399431 18.3384735636 50.0936501016 11.6077134241 65.570045691 23.7657582226 39.8783046498 18.3249292157 +61.8564651063 71.7927431642 43.8070319612 11.3189448865 43.046183241 18.4025593948 49.7436 17.5641 51.2638712274 17.1623497098 52.2038939272 10.4956369212 54.222845621 19.503665544 56.3803054046 14.7884597101 50.7597410528 18.0893946004 43.5951158622 19.9942009773 49.9341271714 22.2561397401 38.4028368753 19.0468586722 36.4754991726 17.7189477884 +39.3237560262 59.3008196656 57.8178546865 10.9451424306 52.0817555582 16.4538745312 47.4359 16.0256 51.2509196468 17.2404558616 50.7451622429 10.5249055723 49.0187553996 23.4892740197 54.8635045999 14.009611915 42.53480844 10.1698684594 50.3399696652 26.4713966117 50.2888186122 13.436410466 37.8323600164 14.4694894463 37.1030736419 15.4456116477 +42.1856791429 66.0348978235 50.9404926595 9.69154713447 42.8790365154 18.4894110252 44.8718 13.718 50.7851572052 17.1801964842 42.7131936166 10.4875472428 35.6535812004 17.5145080595 45.4998576493 3.98542629287 41.5303978512 13.3962907452 40.7489802561 16.1821416622 50.2879944267 12.8722822086 36.9041611715 13.5838157511 34.1489682077 13.5238698677 +30.8469189898 37.3416401041 63.4973230835 11.9140691707 51.5669948027 16.4857136372 48.7179 14.8718 50.5013965822 17.1057707188 45.3172255098 10.4958733944 46.965785757 16.7892016247 49.9367453057 21.5900969196 46.7145179915 15.6132744651 38.3865255803 14.5802151464 50.2347207 20.4044729196 36.286143052 17.1057707266 37.5794219113 15.0158901333 +29.3462004281 42.1487418312 50.0164829541 11.9338520864 58.8391251216 16.6376352154 51.2821 14.8718 50.7336745372 16.9929634144 53.116915041 10.5068605973 49.5842392555 16.7579160147 53.8098729164 17.7664144512 48.4506176492 16.2122782224 38.4040152085 14.4519484496 50.2797016367 15.608737571 62.7866325947 13.9189931024 41.0643057894 12.2144689321 +82.1105579783 1.21055166293 58.6367650809 11.9747210652 60.7557891374 17.0332315064 54.1026 14.8718 50.7113785398 17.0883158538 57.049801823 10.5081369414 51.8562999655 21.969084007 59.8526782804 9.03994992133 59.6326101247 7.05803719901 38.7642788913 14.3655904744 50.2530067222 11.3020842468 66.8176792234 11.4124971575 42.195266147 20.9193721857 +38.3020058088 60.0177857932 54.7302890863 11.4128826732 54.7611561181 16.3661890954 56.1538 14.1026 50.8127448984 16.5727180506 54.8546309465 10.4639151924 55.9528886329 16.7897576048 48.1756567942 19.5212153543 61.6185682473 13.4108227184 41.4701423265 17.2780334445 50.4741411162 12.4619966676 66.755021412 18.0853051378 36.6055782383 15.748875885 +56.5841530218 70.512514809 65.8755478023 11.7324363627 53.5699050003 16.3397125506 52.0513 12.5641 51.014232955 17.2210955293 54.9248425638 10.498331942 51.6637862607 19.9251251977 50.7417190667 17.8384056087 48.8339891275 17.7259558674 47.1554048146 22.3779325323 50.1084399725 18.2590443442 65.4155283864 10.4635122068 39.4311612098 19.3238407541 +33.3393742865 0.5091067352 57.0609827131 9.92056085444 48.9146182416 16.7986370826 48.7179 11.0256 50.3535214052 17.0647430794 63.3095540306 10.4888142392 45.8596693166 17.1289036754 58.4597612975 14.1486628546 45.3426531889 13.9088123935 39.5825667453 17.6484528361 49.919058249 3.7168538855 36.9463314417 13.5143774996 37.5620763361 11.9284457044 +78.7742390407 35.4841012146 46.819907946 10.4946526804 52.6461373069 16.3848363836 47.1795 9.8718 50.4355295722 17.0651685025 49.7494984129 10.4858029097 39.7403795202 18.7410514146 50.1145975193 1.74146171252 42.4150122932 11.6937810961 41.7402438167 17.8293243101 50.0836645085 10.2358819024 37.8254347391 9.60103429534 34.2381100444 13.7131582305 +27.9226442446 25.9868781844 38.3593948725 13.431322624 49.8384929905 16.6763302777 46.1538 6.0256 50.6309819633 17.0765223454 41.1693104232 10.4973002433 52.8112216438 17.8549881176 45.4203820755 5.37240899542 58.9347650647 9.36920121911 39.3118718889 15.6407169743 50.3522608355 12.4686005273 36.722837943 9.33330210686 34.1409410386 4.57766135024 +71.6978651182 10.8681445111 47.3154157773 12.853451784 52.6840664009 16.4079627826 50.5128 9.4872 51.0668072036 17.2088597095 46.6883404976 10.5075321353 56.7891043639 17.3061155207 51.3532681954 0.30387242055 60.6836260407 10.6326415569 41.6798476925 17.7459190133 50.4975722411 7.40962523419 67.0733217806 6.04921458162 36.6592676972 17.6819644553 +74.1383313856 49.1739189791 55.0519165396 11.9499886181 65.8072190346 18.1407649473 53.8462 10.2564 50.7923547263 17.2042143387 52.4189219215 10.4959063596 54.1145317806 17.6735343896 53.091704424 19.2900304453 54.8117983683 6.56964870325 39.0874644519 15.1223039378 50.0163058346 16.2536847911 64.601818107 12.0019169643 40.6437192795 20.2408613716 +32.579020066 1.80811559665 50.5159602579 9.76559162187 60.6739209233 17.0518023644 57.4359 10.2564 50.5512780561 17.0846551833 50.8784883796 10.5030898746 55.571119732 17.369574581 58.9431665444 11.6128777918 61.5868424253 11.0275858356 41.4815028637 18.0474374446 50.4682212165 13.3567024555 65.4372821806 15.5453860785 36.7924815919 13.3921347304 +59.83218542 69.1525081443 49.6774146514 10.3831325083 63.394712866 17.7005351354 60.0 10.641 50.5597580562 17.0938837744 57.0533061394 10.5140692392 65.3064624975 18.1973548277 59.3109904219 10.5721363739 59.7160077628 11.5190968617 77.6060865492 15.1628725383 69.5506789842 9.03298756904 67.0040223792 15.3458266393 75.9483006427 22.7447959146 +35.0306285457 12.5366493416 67.2806595188 14.1286515323 64.8020094467 18.2168786335 64.1026 10.641 75.3259785471 15.7718919896 70.7864511597 10.5136950409 66.9907491546 17.2228706509 68.2221986595 1.13388036584 58.4163840114 5.64577748328 75.9826615205 16.3069223786 69.5186727475 15.1804864298 66.7241920702 5.24980548075 76.8406924717 20.9185472197 +74.3001198284 42.4770945921 66.1730182606 12.0379170217 65.0259398087 18.2878245002 66.9231 10.641 75.04472578 17.0042622562 56.7721495656 10.5032723508 72.1567189391 16.8142747914 62.5117719862 7.49123248279 59.243809528 10.5401420943 76.9457572424 15.858478334 69.5034574098 16.8783132661 68.3076187632 13.2809165227 81.9504776395 19.8745582085 +63.2501970628 65.9524861966 61.0885441374 10.0845344144 65.7555515404 18.5162158303 71.2821 10.641 75.2870877232 16.1749349097 75.4772220231 10.5185680355 70.9805804021 17.15575228 69.3800563617 1.48813233299 56.5943646385 8.91734404523 77.5437200743 15.253949149 69.4994868291 10.2078266798 68.7680482759 13.5214565521 81.6437056853 16.5917599845 +34.1730737648 25.6936743092 66.0530857731 13.3802260073 69.9677341245 20.5389887758 74.359 10.641 75.2399699777 17.0318474866 76.7659121498 10.496999064 72.2220822073 19.3692816955 82.8002478704 18.5293762368 68.4925132247 17.9873634289 77.5847398447 15.8300393854 69.5275296829 9.99229666551 74.1672717461 5.34988087254 84.6102197758 14.2477619017 +40.9138319319 38.5590195509 72.6699892655 15.2342259409 68.8927840241 20.0324488429 78.2051 10.641 75.1524592023 17.0049424015 77.621049129 10.4958742318 79.7001583481 20.740218365 85.1603321777 15.0183695439 65.0059431717 15.9126686464 76.8223042562 15.5951653195 69.5190952617 11.4226857018 64.9003579699 16.2452583689 84.1028794336 8.98065579006 +62.8332930874 62.5108942269 61.503472503 10.8284144754 61.8265465579 17.2696554452 67.9487 8.718 75.9618400861 16.6948422289 65.4144870523 10.6818401251 64.2491606461 14.9139624607 67.095847697 0.601490941965 63.9210367078 15.6209610332 77.3485716647 15.7745292395 69.5618994268 15.9998037858 68.7634335463 8.70057294385 80.18002545 19.0888441297 +42.4767923803 56.7312899691 68.995028632 13.9943105293 60.7879700808 17.0462048583 68.4615 5.2564 75.4480625064 17.0451494125 69.9647805002 10.5072329326 66.9432914334 20.1891325034 62.307750573 5.74997471945 53.5129705438 5.87510952836 77.5731526867 14.7806458332 69.5010445197 5.95583353945 66.8169139163 12.2732943361 80.5732729943 8.36563890168 +52.0334562787 64.5666620298 78.2499161658 17.8832409068 61.9156096372 17.2821912228 68.2051 2.9487 75.7593838168 16.9429296545 64.9697382623 10.4821595117 66.9416766634 18.2435093914 59.9704150458 7.9133104735 57.8328062743 9.22238462333 77.9726106776 14.9556987461 69.524635976 2.73476016988 67.3093466467 0.217006270014 79.0043268649 10.6281977654 +48.9070429644 74.2877488252 36.4819805699 15.1627600944 45.0289975331 17.7244473049 37.6923 25.7692 50.378262298 16.9462798113 38.0502407785 30.4079460343 39.6482666468 22.7313663718 39.9116351739 32.1440226145 31.0329174395 28.9281418896 41.5289197586 24.9164251896 30.4995937106 26.0855615691 34.7318293093 19.6017950724 40.0481864651 24.2614879334 +59.8518383854 72.9583909677 50.9677483784 29.6797742875 39.9213628084 19.7018495281 39.4872 25.3846 50.5336350094 17.0195813691 42.2241202189 30.4980371582 37.9497836858 26.5217492344 39.5965009783 32.1361317349 31.3332440932 32.5385694785 43.7225508034 19.0773278036 30.487391709 19.7794703733 33.6744421497 26.09049021 34.7945935378 13.9696834611 +48.9604602016 72.6295257275 91.1910536059 46.6743428399 84.7942778751 55.5686503735 91.2821 41.5385 77.5009073165 50.1669859475 87.6984642949 50.4899365296 86.5043905611 34.8940252874 95.5934163965 33.2341293953 86.4015498466 38.7469331743 79.3260781751 52.9003912897 89.5001804719 31.9789174357 75.6272550236 37.128751949 79.221764441 22.0945914837 +46.8448551673 36.7917136918 55.8637684939 85.3364867573 55.6629587449 83.3564796243 50.0 95.7692 50.6911241884 87.5139604241 52.3603310701 89.7043170068 50.8154853559 78.950838805 52.772288113 99.6134716833 47.4421116354 98.1843018589 56.6639740815 87.9401250067 50.4102716314 98.6283694405 40.6101254225 89.1362398699 36.0308797708 93.1217332374 +39.9630215796 42.9449148487 49.2805947987 84.0488228306 50.4922475132 78.997532065 47.9487 95.0 49.9903949511 83.997356924 51.4067867898 89.7121385499 51.1834666638 85.1288546497 53.1786144535 98.6038611855 46.2647413368 94.1161921613 57.8217892319 90.6931665489 50.3259243927 94.9946308528 39.1143664549 96.4817513202 34.4995583102 86.6099850511 +66.704943997 32.0150954299 43.3685015447 84.3321772039 51.4671011416 79.2018445773 44.1026 92.6923 50.1271820272 82.990749996 43.588468468 89.7455365536 40.8638552293 82.9261452983 37.6270534612 93.060672952 40.1638157437 87.4486723469 58.2431718991 92.1043278652 50.1040315041 95.0885380179 34.5838289299 89.5889019877 31.1068665551 89.4616352367 diff --git a/example/neural_spline_flow/data/data_proc.jl b/example/neural_spline_flow/data/data_proc.jl new file mode 100644 index 00000000..8d572ca1 --- /dev/null +++ b/example/neural_spline_flow/data/data_proc.jl @@ -0,0 +1,7 @@ +using CSV, DataFrames + +# Read the .tsv file +dat = CSV.read("example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv", DataFrame) +dat_dino = permutedims(hcat(dat.dino, dat.dino_1)[2:end, :]) +dat_dino = map(x -> parse(Float32, x), dat_dino) +dat = vcat(dat_dino, dat_dino) diff --git a/example/planar_radial_flow/planar_flow.jl b/example/neural_spline_flow/main.jl similarity index 55% rename from example/planar_radial_flow/planar_flow.jl rename to example/neural_spline_flow/main.jl index d1ccff81..76abee5c 100644 --- a/example/planar_radial_flow/planar_flow.jl +++ b/example/neural_spline_flow/main.jl @@ -6,51 +6,51 @@ using NormalizingFlows using Zygote using Flux: f32 using Plots + +const NF = NormalizingFlows include("../common.jl") +include("nsf_layer.jl") Random.seed!(123) rng = Random.default_rng() ###################################### -# 2d Banana as the target distribution +# load dataset ###################################### include("../targets/banana.jl") - -# create target p p = Banana(2, 1.0f-1, 100.0f0) logp = Base.Fix1(logpdf, p) -###################################### -# learn the target using planar flow -###################################### -function create_planar_flow(n_layers::Int, q₀) - d = length(q₀) - Ls = [f32(PlanarLayer(d)) for _ in 1:n_layers] - ts = fchain(Ls) - return transformed(q₀, ts) -end - -# create a 10-layer planar flow -flow = create_planar_flow(20, MvNormal(zeros(Float32, 2), I)) +d = 2 +hdims = 10 +K = 8 +B = 3 +Ls = [ + NeuralSplineLayer(d, hdims, K, B, [1]) ∘ NeuralSplineLayer(d, hdims, K, B, [2]) for + i in 1:2 +] +q0 = MvNormal(zeros(Float32, d), I) +flow = Bijectors.transformed(q0, ∘(Ls...)) +# flow = Bijectors.transformed(q0, trans) flow_untrained = deepcopy(flow) -# train the flow sample_per_iter = 10 cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) +checkconv(iter, stat, re, θ, st) = stat.gradient_norm < 1e-3 flow_trained, stats, _ = train_flow( elbo, flow, logp, sample_per_iter; max_iters=200_00, - optimiser=Optimisers.ADAM(), + optimiser=Optimisers.Adam(1e-2), callback=cb, ADbackend=AutoZygote(), + hasconverged=checkconv, ) losses = map(x -> x.loss, stats) ###################################### -# evaluate trained flow +# evaluate trained flow (one can see that the performance is better than the plannar flow) ###################################### -plot(losses; label="Loss", linewidth=2) # plot the loss -compare_trained_and_untrained_flow(flow_trained, flow_untrained, p, 1000) +plot(losses; label="Loss", linewidth=2) # plot the loss \ No newline at end of file diff --git a/example/neural_spline_flow/nsf_layer.jl b/example/neural_spline_flow/nsf_layer.jl index 6820f48f..2353c634 100644 --- a/example/neural_spline_flow/nsf_layer.jl +++ b/example/neural_spline_flow/nsf_layer.jl @@ -10,79 +10,107 @@ Neural Rational quadratic Spline layer # References [1] Durkan, C., Bekasov, A., Murray, I., & Papamakarios, G., Neural Spline Flows, CoRR, arXiv:1906.04032 [stat.ML], (2019). """ -struct NeuralSplineLayer{T1,T2,A<:AbstractVecOrMat{T1}} <: Bijectors.Bijector +# struct NeuralSplineLayer{T1,T2,A<:AbstractVecOrMat{T1}} <: Bijectors.Bijector +# dim::Int +# mask::Bijectors.PartitionMask +# w::A # width +# h::A # height +# d::A # derivative of the knots +# B::T2 # bound of the knots +# end + +# function NeuralSplineLayer( +# dim::Int, # dimension of input +# hdims::Int, # dimension of hidden units for s and t +# K::Int, # number of knots +# B::T2, # bound of the knots +# mask_idx::AbstractVector{<:Int}, # index of dimensione that one wants to apply transformations on +# ) where {T2<:Real} +# num_of_transformed_dims = length(mask_idx) +# input_dims = dim - num_of_transformed_dims +# w = fill(MLP_3layer(input_dims, hdims, K), num_of_transformed_dims) +# h = fill(MLP_3layer(input_dims, hdims, K), num_of_transformed_dims) +# d = fill(MLP_3layer(input_dims, hdims, K - 1), num_of_transformed_dims) +# mask = Bijectors.PartitionMask(dim, mask_idx) +# return NeuralSplineLayer(dim, mask, w, h, d, B) +# end + +# @functor NeuralSplineLayer (w, h, d) + +# # define forward and inverse transformation +# function instantiate_rqs(nsl::NeuralSplineLayer, x::AbstractVector) +# # instantiate rqs knots and derivatives +# ws = permutedims(reduce(hcat, [w(x) for w in nsl.w])) +# hs = permutedims(reduce(hcat, [h(x) for h in nsl.h])) +# ds = permutedims(reduce(hcat, [d(x) for d in nsl.d])) +# return Bijectors.RationalQuadraticSpline(ws, hs, ds, nsl.B) +# end + +## Question: which one is better, the struct below or the struct above? +struct NeuralSplineLayer{T,A<:Flux.Chain} <: Bijectors.Bijector dim::Int + K::Int + nn::AbstractVector{A} # networks that parmaterize the knots and derivatives + B::T # bound of the knots mask::Bijectors.PartitionMask - w::A # width - h::A # height - d::A # derivative of the knots - B::T2 # bound of the knots end function NeuralSplineLayer( - dim::Int, # dimension of input - hdims::Int, # dimension of hidden units for s and t - K::Int, # number of knots - mask_idx::AbstractVector{<:Int}, # index of dimensione that one wants to apply transformations on + dim::T1, # dimension of input + hdims::T1, # dimension of hidden units for s and t + K::T1, # number of knots B::T2, # bound of the knots -) where {T2<:Real} + mask_idx::AbstractVector{<:Int}, # index of dimensione that one wants to apply transformations on +) where {T1<:Int,T2<:Real} num_of_transformed_dims = length(mask_idx) input_dims = dim - num_of_transformed_dims - w = [MLP_3layer(input_dims, hdims, K) for i in 1:num_of_transformed_dims] - h = [MLP_3layer(input_dims, hdims, K) for i in 1:num_of_transformed_dims] - d = [MLP_3layer(input_dims, hdims, K - 1) for i in 1:num_of_transformed_dims] + nn = fill(MLP_3layer(input_dims, hdims, 3K - 1), num_of_transformed_dims) mask = Bijectors.PartitionMask(dim, mask_idx) - return NeuralSplineLayer(dim, mask, w, h, d, B) + return NeuralSplineLayer(dim, K, nn, B, mask) end -@functor NeuralSplineLayer (w, h, d) +@functor NeuralSplineLayer (nn,) # define forward and inverse transformation -function instantiate_rqs(nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector) +function instantiate_rqs(nsl::NeuralSplineLayer, x::AbstractVector) # instantiate rqs knots and derivatives - ws = permutedims(reduce(hcat, [w(x) for w in nsl.w])) - hs = permutedims(reduce(hcat, [h(x) for h in nsl.h])) - ds = permutedims(reduce(hcat, [d(x) for d in nsl.d])) - return Bijectors.RationalQuadraticSpline(ws, hs, ds, nsl.B) + T = permutedims(reduce(hcat, map(nn -> nn(x), nsl.nn))) + K, B = nsl.K, nsl.B + ws = T[:, 1:K] + hs = T[:, (K + 1):(2K)] + ds = T[:, (2K + 1):(3K - 1)] + return Bijectors.RationalQuadraticSpline(ws, hs, ds, B) end -function Bijectors.transform( - nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector -) +function Bijectors.transform(nsl::NeuralSplineLayer, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) # instantiate rqs knots and derivatives rqs = instantiate_rqs(nsl, x_2) - y_1 = transform(rqs, x_1) + y_1 = Bijectors.transform(rqs, x_1) return Bijectors.combine(nsl.mask, y_1, x_2, x_3) end -function Bijectors.transform( - insl::Inverse{<:NeuralSplineLayer{<:Vector{<:Flux.Chain}}}, y::AbstractVector -) +function Bijectors.transform(insl::Inverse{<:NeuralSplineLayer}, y::AbstractVector) nsl = insl.orig y1, y2, y3 = partition(nsl.mask, y) rqs = instantiate_rqs(nsl, y2) - x1 = transform(Inverse(rqs), y1) + x1 = Bijectors.transform(Inverse(rqs), y1) return Bijectors.combine(nsl.mask, x1, y2, y3) end -function (nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}})(x::AbstractVector) +function (nsl::NeuralSplineLayer)(x::AbstractVector) return Bijectors.transform(nsl, x) end # define logabsdetjac -function Bijectors.logabsdetjac( - nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector -) +function Bijectors.logabsdetjac(nsl::NeuralSplineLayer, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) rqs = instantiate_rqs(nsl, x_2) logjac = logabsdetjac(rqs, x_1) return logjac end -function Bijectors.logabsdetjac( - insl::Inverse{<:NeuralSplineLayer{<:Vector{<:Flux.Chain}}}, y::AbstractVector -) +function Bijectors.logabsdetjac(insl::Inverse{<:NeuralSplineLayer}, y::AbstractVector) nsl = insl.orig y1, y2, y3 = partition(nsl.mask, y) rqs = instantiate_rqs(nsl, y2) @@ -90,9 +118,7 @@ function Bijectors.logabsdetjac( return logjac end -function Bijectors.with_logabsdet_jacobian( - nsl::NeuralSplineLayer{<:Vector{<:Flux.Chain}}, x::AbstractVector -) +function Bijectors.with_logabsdet_jacobian(nsl::NeuralSplineLayer, x::AbstractVector) x_1, x_2, x_3 = Bijectors.partition(nsl.mask, x) rqs = instantiate_rqs(nsl, x_2) y_1, logjac = with_logabsdet_jacobian(rqs, x_1) diff --git a/example/planar_radial_flow/radial_flow.jl b/example/planar_radial_flow/radial_flow.jl deleted file mode 100644 index 4d51cd24..00000000 --- a/example/planar_radial_flow/radial_flow.jl +++ /dev/null @@ -1,55 +0,0 @@ -using Random, Distributions, LinearAlgebra, Bijectors -using ADTypes -using Optimisers -using FunctionChains -using NormalizingFlows -using Zygote -using Flux: f32 -using Plots -include("../common.jl") - -Random.seed!(123) -rng = Random.default_rng() - -###################################### -# 2d Banana as the target distribution -###################################### -include("../targets/banana.jl") - -# create target p -p = Banana(2, 1.0f-1, 100.0f0) -logp = Base.Fix1(logpdf, p) - -###################################### -# learn the target using radial flow -###################################### -function create_radial_flow(n_layers::Int, q₀) - d = length(q₀) - Ls = [f32(RadialLayer(d)) for _ in 1:n_layers] - ts = fchain(Ls) - return transformed(q₀, ts) -end - -# create a 20-layer radial flow -flow = create_radial_flow(10, MvNormal(zeros(Float32, 2), I)) -flow_untrained = deepcopy(flow) - -# train the flow -sample_per_iter = 10 -cb(iter, opt_stats, re, θ) = (sample_per_iter=sample_per_iter,) -flow_trained, stats, _ = train_flow( - elbo, - flow, - logp, - sample_per_iter; - max_iters=200_00, - optimiser=Optimisers.ADAM(), - callback=cb, -) -losses = map(x -> x.loss, stats) - -###################################### -# evaluate trained flow -###################################### -plot(losses; label="Loss", linewidth=2) # plot the loss -compare_trained_and_untrained_flow(flow_trained, flow_untrained, p, 1000; legend=:bottom) From 30cfc329a6167788e3c3b4851fda78bb0b1275fb Mon Sep 17 00:00:00 2001 From: Zuheng Date: Thu, 17 Aug 2023 02:00:12 -0700 Subject: [PATCH 38/40] rm unused data in nsf lfow --- .../data/DatasaurusDozen-Wide.tsv | 144 ------------------ example/neural_spline_flow/data/data_proc.jl | 7 - 2 files changed, 151 deletions(-) delete mode 100644 example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv delete mode 100644 example/neural_spline_flow/data/data_proc.jl diff --git a/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv deleted file mode 100644 index 7f8dda94..00000000 --- a/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv +++ /dev/null @@ -1,144 +0,0 @@ -away away bullseye bullseye circle circle dino dino dots dots h_lines h_lines high_lines high_lines slant_down slant_down slant_up slant_up star star v_lines v_lines wide_lines wide_lines x_shape x_shape -x y x y x y x y x y x y x y x y x y x y x y x y x y -32.3311102266 61.411101248 51.2038911373 83.3397766053 55.9930301513 79.2772636977 55.3846 97.1795 51.1479167122 90.8674123313 53.366566866 90.2080300059 57.6132335477 83.9051707998 52.8720214902 97.3432226575 47.6952008891 95.2411869282 58.2136082599 91.881891513 50.4815081703 93.2227013657 65.8155400946 95.5883740582 38.3377571839 92.472719051 -53.4214628807 26.1868803879 58.9744699018 85.499817612 50.0322537946 79.013071199 51.5385 96.0256 50.5171258092 89.1023945866 52.8019793617 90.0880645063 51.2743918205 82.8179829379 59.0141444945 93.5748748733 44.6099759113 93.075835032 58.1960536923 92.2149886482 50.2824056687 97.6099835723 65.6722651618 91.9334018119 35.7518707905 94.1167680276 -63.92020226 30.8321939163 51.8720726696 85.8297376348 51.2884586593 82.4359398425 46.1538 94.4872 50.2074801993 85.4600473951 47.0541298828 90.458936026 50.7538984114 76.7541289533 56.3751090389 96.30514763 43.8563814822 94.0858717713 58.7182307185 90.310532087 50.1867033389 99.6946801425 39.002716451 92.26183816 32.7672179591 88.5182945794 -70.2895057187 82.5336485877 48.1799307865 85.0451167372 51.1705368873 79.1652941091 42.8205 91.4103 50.0694819181 83.0576695262 42.4484337816 89.5077001153 37.0211840116 81.954469997 37.8391996844 94.359443503 41.5789293984 90.3035665387 57.27837287 89.9076067166 50.3269108629 90.0220534916 37.7953028026 93.5324553607 33.7296067755 88.622265789 -34.1188302357 45.7345513203 41.6832004024 84.0179406019 44.3779145287 78.1646280036 40.7692 88.3333 50.5628463386 82.9378217764 42.7040363241 90.4426288607 42.8817550897 80.1847700514 39.8853734176 90.6346620817 49.1774185559 96.6105324538 58.082020494 92.0081450101 50.4562073315 89.9874101286 35.5139005857 89.5991901138 37.238249327 83.7249284144 -67.6707164012 37.110947969 37.8904155015 82.5674929978 45.0102700721 77.8808631153 38.7179 84.8718 50.288527801 82.9752535659 32.3789386326 90.144142631 37.1557104777 84.9541114129 44.0774051326 84.1258533023 42.6522509813 90.5606397307 57.4894477748 88.0852855629 30.4648469308 82.0892320845 39.2194529062 83.5434818364 36.0271982243 82.0407806505 -53.2591294055 97.4757710964 39.5489736947 80.812601774 48.559816719 78.7883706035 35.641 79.8718 25.5834750822 82.9148911324 32.5307027362 70.1573965085 38.7318581261 83.7893564281 31.4970254888 67.9125621568 30.1233318217 81.1442953304 28.088741319 63.5107944263 30.5015964737 82.3071263462 31.5882013862 77.0280524208 39.2392807768 79.2637238398 -63.5149808671 25.1000785788 39.6495738769 82.6645338713 42.1422682922 76.8806345544 33.0769 77.5641 25.4835833896 82.929084979 33.3652644305 70.4593354392 31.0005254108 82.5794805962 25.9525965456 73.0331803541 31.915519986 79.7904134869 28.0854682136 63.5901969481 30.489569227 77.7298801366 32.6577782482 80.1138612983 39.7845249391 82.2605658962 -67.9805388133 80.9571652197 34.7505970472 80.0110909897 41.0269715735 76.4095884613 28.9744 74.4872 25.4435257049 82.8742004973 32.6560991775 70.0510704992 25.9854996226 74.3493875478 36.4347239148 62.2940454639 27.8877193574 75.4455759934 28.087273053 63.1232828116 30.45024833 79.8348944011 31.6415992679 77.6108331874 35.1660293896 84.1564919839 -67.3724659005 29.720400203 27.560835291 72.8478255927 34.5753090115 72.7248411845 26.1538 71.4103 25.5651134237 82.9240977704 22.9550932697 70.4268951523 23.7145662909 75.0980490792 24.997482691 75.4415616626 27.5482344526 75.1647590544 27.5780252176 62.8210386566 30.5183947985 68.2525829388 30.9871426099 63.9440280336 40.622115916 78.5421042089 -15.5607495229 80.0656402858 24.6355399754 71.6107148256 31.1686007236 69.2454213193 23.0769 66.4103 25.9288442731 82.8211841087 27.1498436318 70.2108058428 23.077319507 71.7205396747 32.2462751352 70.1105868383 20.2097781637 67.5166420988 27.7799191139 63.518147517 30.5116612138 66.0937181831 27.4396322054 65.7444629564 39.181907086 79.8190371976 -71.7907676942 71.0654666627 20.9594648125 66.0414983826 32.6442540714 70.7325550334 22.3077 61.7949 27.5514782612 51.4873865306 26.1669092779 70.5003151212 24.9391368744 72.5802858084 18.1094722949 53.7081103324 24.4434528292 54.4736551228 28.5889998149 63.0240805653 30.4980777147 62.3849850338 30.5783699449 65.5219454667 42.4308889944 75.1336346406 -70.2425464362 24.1095975542 20.6891490538 62.7213052064 26.6666456539 62.9809712052 22.3077 57.1795 27.5304663693 51.4148465629 26.3314570795 50.494556525 17.8934987136 71.1096873431 21.7960992263 49.4995295248 24.6308301331 53.3957388998 28.7391414969 62.7208638859 30.6202277519 60.8600821544 28.809401637 59.6210044666 43.0812558617 75.3467016413 -64.9374355444 81.5542049945 19.2882047369 62.0630593582 26.4592623274 62.6346733709 23.3333 52.9487 27.0955703605 52.0767994399 22.0037091415 50.4994401238 23.6730818572 64.8708391385 24.150491069 47.1885335266 21.0379688969 49.829299645 27.0246032407 62.9018588649 30.4921854231 55.8789168923 27.9395185786 50.9850621313 44.3607194344 70.4742070546 -62.2135245453 21.4758389969 20.0245005746 61.3426238695 25.5380076465 60.6302361093 25.8974 51.0256 27.4392496133 51.7120790527 23.2847479782 50.4743528321 23.7490719056 63.557171895 28.6031956015 44.2237227 27.106242932 54.7195531488 28.8013366963 63.389040388 30.474634333 52.3221596388 33.8499497887 46.2844744465 44.3247788435 71.0459044019 -67.2694004772 18.7089683725 35.4695229971 43.1158849517 26.3207111933 62.3962811186 29.4872 51.0256 27.8782642594 50.70890793 27.6340383923 50.5036668574 32.2151786343 65.6009580191 20.9319996761 51.6462354868 28.0896176553 60.512212549 27.186463838 63.5587296478 30.500998079 48.0696996767 32.628463049 54.0637204929 47.552160202 66.4604037217 -40.5701970446 79.3729634752 36.8943276507 47.7065559687 26.1633874648 62.0268148985 32.8205 51.0256 27.3388689181 51.6530467544 32.0371080007 50.5000661229 29.4368412489 67.5545281141 35.4355325251 34.9610381566 31.1357349152 61.4319127116 29.2851466002 63.3836058254 30.4944213623 45.7734755346 32.4386500903 49.9030194916 48.6649246215 64.9610049438 -74.7411813341 21.1016372041 39.0555497751 55.5469737131 25.2137904611 60.2733986018 35.3846 51.4103 27.6784029741 51.1819891691 29.3365106528 50.461893969 40.0500951983 71.8890335708 44.3773828236 57.6847369215 37.700837521 68.4184052804 39.4029453026 51.1508571967 30.4995445502 49.3820997506 35.8382920833 53.3514883396 46.280631958 68.0284231525 -71.7683189223 20.0110618423 46.9570801494 65.2404073935 26.8595295604 63.1418346727 40.2564 51.4103 52.6356576791 51.4185522584 40.816953133 50.4877601957 37.5747950188 69.9887444412 46.313692406 55.6855917703 43.3926884016 43.5975940692 28.8113284396 61.3578540576 50.144392979 47.0142836417 37.1420429198 49.7419879519 50.5632444849 62.6238279659 -76.1669198143 75.9361704048 37.3104527352 45.2587508999 31.7606691716 69.7976553936 44.1026 52.9487 52.0252141055 52.1230110467 42.7390751173 50.483155992 49.3104686263 67.1687603206 46.3860739231 51.0021645381 51.2623974761 49.6665051258 34.3039579069 56.5421259093 49.9128985877 55.5878389121 34.6445467746 54.4441077764 52.6309642318 59.9639357915 -65.6236574431 15.5828033531 40.0096720019 60.9865825077 39.7932905744 75.8079642168 46.6667 54.1026 52.8811647904 50.6215547554 51.8678937691 50.4932732621 47.3480185524 72.5916115408 42.4954400876 56.9036815315 43.2867129203 43.0568603208 29.6027609801 60.157346722 50.3374274859 57.9553142818 35.675404696 49.6256433471 54.6362064446 57.4938424367 -50.8506336394 13.9876016304 48.0143866846 65.7128195934 45.6174391854 78.1199702277 50.0 55.2564 52.9526073133 50.0747390118 43.3710018924 50.5008373234 50.157414434 71.1617258246 44.3005150328 58.1329592781 55.9059162338 54.2526725342 49.1161568603 63.6600006211 50.4189791992 60.5249117346 64.99458547 49.4621728312 51.346027139 61.6005530364 -33.0240700249 24.4678303872 53.7037759268 66.3894820363 53.7714606827 79.3029316435 53.0769 55.641 52.5205524936 51.5024421035 53.0427983006 50.4875038691 58.2100082697 68.0445380719 46.9882833458 55.4206367313 50.198958949 52.2250295948 39.6175458263 62.9251879574 50.2052589714 57.9242171676 66.1291498342 61.6437178853 57.134855354 53.8779658045 -39.7063261674 84.2752871038 63.0674998921 64.0350004581 56.4514336733 79.2471070208 56.6667 56.0256 52.3428220582 51.8619520908 56.6189102967 50.5033740178 56.6089076561 70.4089460888 52.4215231623 47.867005488 55.761587372 54.0305172396 43.23308466 63.1652187223 50.4353151633 56.4393224734 65.790651882 55.9453737171 54.1712415737 58.0598078952 -45.5964849542 9.76334884943 62.0480325076 63.8458632494 66.0937201989 77.0178182606 59.2308 57.9487 51.9275902141 52.2577906065 70.568289819 50.4962361243 52.8338205303 72.5518862725 64.090998189 68.6509850107 58.3231396505 60.0310591646 64.8927879422 65.8141767574 69.5057889042 59.2849248279 67.5017806138 60.8502274659 50.9423823071 62.0973939351 -42.9680469104 17.9454583961 59.8399667124 64.4767644437 56.9258394194 79.2063186226 61.2821 62.1795 52.7137744894 51.1979443188 67.870887292 70.0680261345 61.8185854576 76.1128240228 63.0168717563 68.2252589114 70.171457528 70.2822212385 62.4901493154 74.5842896072 69.5340709833 64.8929773346 67.6295142176 61.4291060701 66.512924456 59.0712818478 -52.4944067819 16.0511142003 55.1609418175 65.2373081694 58.9882538467 78.9172589554 61.5385 66.4103 50.4438027888 82.9418288173 58.8215465891 70.4557817483 61.7276334143 77.0692384058 67.3025779226 64.4134627173 60.600276471 59.7533088802 68.9880844294 63.232147305 69.5101069302 61.3296244282 63.9100640599 62.9723349305 64.3979095023 56.1748754008 -46.0822757831 23.1104578154 61.2797865792 65.766402504 57.9022660533 79.0808678187 61.7949 69.1026 50.2166950326 83.7523429676 60.1869278445 70.4835512458 62.1922520815 76.1568192103 67.2259568904 62.4764105301 63.7602750188 61.1856411696 62.1056186306 75.9908707599 69.5263228876 56.9952791254 66.5493579687 67.7255886599 68.686593294 62.2373439902 -74.2477082092 20.314187812 60.8349175348 64.6037697063 64.8128669568 77.4869389235 57.4359 55.2564 52.1841801076 51.9752506714 59.4355174849 50.4206326073 58.1187658587 73.1189614684 50.309988715 47.2678818553 54.3894952619 53.811901225 32.4618467439 62.8819029189 49.9472399619 48.2295767261 64.0515417112 48.8200939835 65.0277937057 57.0962573525 -64.5682641863 83.6396338956 61.5205906458 64.7918550418 60.3497935166 78.7294458673 54.8718 49.8718 52.7920973459 51.0733956509 49.5858434795 50.4349813825 58.2704123843 35.5968950425 45.4126389249 53.8707976895 55.2412969947 52.5935746267 41.3272006535 49.0702512739 50.0631041291 58.085037556 68.1932895833 46.5435408096 53.3045031712 40.883844549 -74.0216939058 76.1282745076 36.916543863 41.0952474383 48.890555756 16.8065435154 52.5641 46.0256 52.5897198565 51.338090196 41.7797417848 50.4706422808 59.7665303129 32.6212516009 54.0206071815 47.2698972057 49.0701267502 48.8969446811 44.0071499323 46.4496737846 50.489902373 39.2054119682 65.2783891952 44.8203529292 52.9451780881 40.3846256942 -62.3911805626 5.62307076073 38.5021996714 41.5671556174 47.549506348 17.0714045245 48.2051 38.3333 52.0288486671 52.1768375002 46.1276863394 30.4494107792 52.5355347207 27.1741229419 49.8420222483 50.5765522936 46.0181190216 46.2577233718 44.0740606888 34.5532038906 49.6888638379 42.018514414 34.9976613883 28.6329939351 45.225738199 30.5552259226 -74.189036683 68.1335832223 48.6643707344 30.6806668519 30.8894308257 26.3290372348 49.4872 42.1795 52.729246581 51.2017650475 38.0337837705 50.4908995189 50.8925215809 29.1452839382 50.7132995657 48.7722867825 44.7595874356 41.9746066134 44.0013152386 33.9042073544 50.3334811097 43.588671479 33.9237661692 53.5586885227 50.8433046202 37.7446280981 -28.2367819396 56.1395964513 50.2852524042 30.3379221051 28.972835724 29.0504689185 51.0256 44.1026 52.8843132864 50.4414354501 44.5468412844 50.4159133758 49.9675511775 28.4602033343 55.0455549106 45.9862727459 51.6006241332 50.0324668853 45.0063004454 38.2990195507 50.4467514196 57.2305889423 66.0044543688 52.6846691452 49.6592939398 36.273442735 -75.7719387944 69.8292300322 42.276332666 34.5276361236 35.1613607523 22.7868496473 45.3846 36.4103 52.5093008862 51.4162051508 46.2381175975 30.4956129408 51.3478832298 27.3642602259 37.5437012597 32.8840244657 36.4658820182 35.5241876075 44.4438406096 36.0190833012 50.4209091963 43.3479568967 37.0232562464 32.1184816794 47.1427889348 32.0250641734 -75.8552294691 62.5170442862 54.0317756204 29.6723483149 45.8703691685 17.4893933655 42.8205 32.5641 50.8626843321 17.1456310914 49.7397453317 30.4727248868 37.5551162883 24.5376638386 39.7958978066 28.4432191156 33.3947987327 35.0168013145 42.1787133986 26.492119482 50.3740520508 32.9182406167 34.6982700852 36.4753541349 41.2700565259 21.7218171465 -65.9708570175 72.7448559954 37.3293552552 39.6020423074 32.8314206076 24.7975381459 38.7179 31.4103 50.8914922458 17.1413237277 39.1102518936 30.4786951789 39.1995941778 23.6804956588 28.5233910215 39.7966662481 37.5800715749 38.8315722007 44.0445656189 35.6622382764 30.4768299051 39.4246055913 34.8634441715 30.9392745218 47.1301271555 31.3099029222 -21.7780404779 6.61662530728 41.389522551 37.2960562328 39.4855747899 20.0330500489 35.1282 30.2564 25.8551275976 17.0819086886 39.1428377913 30.7928559394 36.3827221955 25.2380052325 31.3996460012 39.2652632229 36.619722059 36.8114141094 41.6404540171 27.0930954213 30.5034743497 38.3485561381 31.0900650789 32.3837547196 40.9159715783 23.5415923929 -67.7597962473 72.4212015285 40.0746666572 34.6236852027 33.4461829314 24.1906434075 32.5641 32.1795 26.0256445518 16.9250135319 34.6254702671 30.4997410297 41.1006943822 26.5526132161 39.2415652162 33.3614166846 36.1634664356 33.819314199 41.9383300069 24.9915229793 30.5048349283 28.7714879939 34.9213061465 35.4184380529 38.6601639737 21.6895496369 -78.6171953363 52.5752573142 35.3496806211 47.1410731332 21.8635812756 46.0525933457 30.0 36.7949 27.8931727187 50.6619634101 34.7542074707 30.4958650002 28.2178118248 28.2802924847 27.7140565712 42.2500516253 33.1677570349 31.2477411672 44.0539275103 33.5563924949 30.4622184647 32.7504720275 30.343361183 32.5245631742 42.6125508262 24.195053774 -68.5077081898 15.4569189652 34.7637004189 47.6247999245 25.1658708449 35.7666139539 33.5897 41.4103 27.6399679354 51.3990974838 36.7537002386 50.5293043035 33.7970720433 28.6997507433 32.9607035213 35.8231112903 42.6449829318 40.1543216386 39.2067193308 51.5337156971 30.491302622 43.5613740753 35.4833288685 44.4916349303 41.2090395008 24.1287091891 -74.8850211598 25.4166063231 37.0266294472 44.4622930512 22.9594789762 51.2877067934 36.6667 45.641 27.892658895 50.7952815163 37.0193512929 50.4926666972 43.9193873055 27.7978445882 35.4047362397 31.9391621952 47.2250252695 44.5756466185 28.7044492315 61.7775254006 30.449653838 44.9951230008 33.2349104271 38.502483721 50.6492211499 37.4837907435 -66.4549036599 19.8366286542 36.4555695327 40.791843032 25.5860552415 60.8465833585 38.2051 49.1026 52.7977329399 50.6860370901 41.4171152753 50.4697734181 44.4164011084 28.9476923345 51.6422264225 50.8383406788 48.3923630032 50.3315285645 31.7086628996 58.837754374 30.4859942157 47.3479464653 35.4399436182 47.7975176939 52.1319019406 39.3646020815 -77.3178020985 48.3983464352 35.5376642131 48.7293868661 26.3143100672 33.2971681038 29.7436 36.0256 27.5806388053 51.5247612607 22.8171074784 30.4360403387 28.0376027706 26.8038243613 34.207620843 33.5938054385 27.2451932001 29.8195074488 42.8117114739 30.0204484197 30.5020767935 36.4412129198 33.8735095227 37.056028453 39.7972712705 16.083549649 -58.9124603193 75.6677562173 20.4089478892 32.2030304205 33.037301251 24.5644624921 29.7436 32.1795 26.4913985349 17.4053909738 34.800701463 30.4979313851 31.6895761296 26.6664002089 36.5234892198 35.6710483891 30.4043143388 29.0795399241 43.300614891 31.5264261979 30.4780588886 28.020580489 28.5641377966 34.6263997171 43.5498814807 24.3072109395 -57.617447817 8.19480060319 23.4957104659 25.3224681542 36.4688894772 21.8761144725 30.0 29.1026 25.9853178159 17.203722132 27.8918897953 30.4849702595 27.2796128523 19.2572809685 28.4192021281 21.3157482481 30.0807992865 27.5880255173 40.3986329069 16.3470083822 30.5011608787 38.3141385582 31.3195486706 31.6639905316 36.7146633371 17.3818098572 -76.0882257967 59.6799300235 29.5575433634 21.3647774591 26.5135531009 32.9187498178 32.0513 26.7949 26.2014192753 17.0938239128 32.8706391587 30.5023417488 31.2218783581 26.4408557812 26.1604969417 23.0422314726 33.9806339675 35.0330581489 40.4356915763 20.2326706762 30.5047730174 26.4166256707 30.796174379 23.5190343602 37.2246101278 17.9800644828 -57.4660505497 1.50441817488 33.0082339121 15.9850714584 36.0270912125 22.1954385395 35.8974 25.2564 25.8575694746 17.113842659 37.7940494081 30.4926383364 33.5458051034 20.0807081014 40.435327397 27.6400711421 34.2506117649 34.4415865654 40.9365466658 16.9130048364 30.509115326 22.8255471196 34.2333554347 22.4959116015 40.5124478537 22.9622145725 -79.4283834934 45.2107942872 53.9803991821 29.3509867092 42.6810404129 18.5425361977 41.0256 25.2564 50.7046843629 17.0237445436 35.6673163042 30.522035054 39.7363306065 23.4948069772 44.544955314 27.3622426221 51.1411835714 20.6183237784 39.6615736653 15.609355577 50.1156321728 16.6940065479 36.6307877065 31.4768312621 40.2220627884 20.6146217044 -76.3565221496 10.4182411281 52.2343086004 29.7116729865 45.0410771273 17.7345177303 44.1026 25.641 50.8119753465 17.1149252621 48.4422995026 30.5017227753 42.8418426525 20.3339004396 40.1237272008 30.0731619402 49.899592066 19.5640835356 40.899259175 20.7985289466 50.4483013392 21.6402564599 34.5486191018 26.1110878654 44.1249612057 23.3662452858 -64.4050752632 78.5841760758 59.5030766063 30.6696739448 47.3524164557 17.1347454264 47.1795 28.718 50.5648455577 17.0777773216 58.6693732631 30.5068378948 47.0937298879 26.9934706312 42.1848064503 28.5736747331 55.5898773424 26.2992451143 41.9686168283 26.4970725985 50.17317743 29.177864393 37.5196356 29.2676058936 42.3462616039 24.5583269437 -40.6350418091 73.3947503698 41.1637810689 34.3157582514 53.8461940322 16.3873180961 49.4872 31.4103 50.9393039127 16.981021883 47.9169554375 30.4867257545 51.4536583539 26.0229417521 40.0026592892 27.4875100027 55.4820563581 22.8274662445 40.3834058238 21.3912255247 49.9227419194 38.4404729124 62.4256973267 34.0442062707 44.1890843708 28.772625473 -43.9498645857 75.9587156671 48.9930401201 32.0303588363 55.1296393835 16.3532468711 51.5385 34.8718 50.4588548392 17.0385789686 51.8941663939 30.494755684 49.3677068167 22.005580703 57.1252310509 42.7390191388 63.5729393821 32.9386176053 56.5381264538 32.4442454697 50.3112993746 26.8604882737 64.320383794 31.7856294846 45.8142453101 31.056979672 -30.9962205791 71.694404938 59.2692803248 29.6407017654 52.1881199617 16.4182174652 53.5897 37.5641 52.9013640729 50.6905627231 51.7490891643 30.4729933399 48.2534809736 29.2708555331 60.375744968 41.3564579961 59.0250770848 28.3220858801 52.9706912828 29.0401966941 50.3994096114 38.9638389234 65.4455325117 43.2734991936 46.6431192691 31.4072190618 -68.2307689907 80.8725016628 45.4691770035 33.8311927323 58.6726630845 16.6633724322 55.1282 40.641 52.6849534438 51.2944692191 58.2465530013 50.4428744949 61.0029597643 30.0153144324 55.1058961252 46.1490510561 62.4237785929 33.4865952868 54.6209525885 30.3445244537 50.3373626448 40.913386144 64.3256315887 41.6159763773 52.1898883047 39.3677964938 -72.0463894612 12.9180067349 62.6912665406 30.2903738312 82.1443116037 33.1952218669 56.6667 42.1795 52.500088943 51.594356165 57.3106968674 50.4825132895 60.8065608049 27.8683134861 61.7372710245 38.5848440052 47.849907703 48.556624645 65.0990443941 27.2415575588 50.2023180965 47.2129480045 65.264338195 35.7175447772 51.1003173587 38.095699984 -46.5927679682 84.9723827774 73.4286708654 48.5778551303 75.6865711383 24.7415103908 59.2308 44.4872 51.8356372562 52.3357655339 67.9680481127 50.4521589097 59.0674270249 26.7882840217 55.7412443246 46.6535769763 52.2685616825 52.8052401961 63.0559909076 29.7090956748 50.2749326091 49.6234993439 66.388704175 38.8447443366 65.4008341477 42.6140320392 -49.2572183396 81.8814032306 70.846426107 52.282253331 85.344964503 52.886419866 62.3077 46.0256 76.9954121013 52.0455286534 65.9493192681 50.494240794 61.4146337175 31.0393750583 58.2964084158 41.5831977346 68.8510215758 39.573434273 70.9601362323 41.2595012867 69.4943803061 52.6124107059 68.1914228192 45.1694864796 64.8520398267 43.1337348404 -42.7817612539 12.9911884302 71.5390198495 45.5218061588 83.8446535127 58.6160602305 64.8718 46.7949 77.310600475 51.7467300374 55.7470928102 50.4789748003 68.9343694692 31.6198930866 57.0425482213 42.418757739 65.6763960251 35.1093154301 69.8958192404 43.4537592655 69.5193397556 47.384533707 68.5406624515 48.4385263389 63.4038060118 44.847493688 -65.475952195 14.2745856444 67.6208658884 38.0065584658 85.66476066 45.5427527692 67.9487 47.9487 77.9260443352 50.3186604237 68.4030100436 50.4986272318 63.0652438592 34.2357465852 76.7756112073 55.2545987009 77.7468851148 47.3718241139 70.5958928563 41.9647438672 69.5291826079 57.8412666804 67.6614312657 51.0685845873 65.3715488709 42.6959713164 -71.9650826544 17.7102359443 72.4709525633 51.1221348241 77.931004934 68.6900618808 70.5128 53.718 77.2543883433 51.4618248152 76.696465628 50.4938305609 72.2725228706 67.6444437851 71.8870772602 59.2669926152 80.352934115 47.8002988852 69.6470214273 44.044445022 69.495755496 58.785073162 68.6755160534 50.465234811 61.9080397046 53.1903520694 -32.1464623358 43.4817094425 64.8122375621 62.8109155873 73.0318330209 73.12057473 71.5385 60.641 76.2431578028 52.1236898503 67.2148371159 70.2600284356 75.3598819664 75.4216145799 73.7883730663 58.6208121164 60.3375864858 63.0349593272 77.392982494 63.3714590552 69.199057706 60.4189573202 69.0817099788 55.4740530987 71.8266529973 65.6862303604 -31.8384976954 71.8121653901 60.8536798727 65.4991470327 70.1028044999 75.6832068542 71.5385 64.4872 77.0844843669 51.9671367003 72.3020057291 70.2204373812 72.8336258709 72.1146938095 75.1396847343 54.2445521781 61.6587233282 64.1000234779 64.4007871926 67.4487184472 69.5510914402 58.4010982686 69.048716493 66.3660457477 72.6546231621 66.8782776113 -31.0052582572 40.682503007 67.7894961571 61.3637015467 57.1446946889 79.1941025922 69.4872 69.4872 75.2280531954 82.9856620201 66.7632573944 70.4738617113 70.8104148392 79.3504618327 64.5963189058 65.952512925 71.5450389858 72.657716331 63.8689598271 70.2137388333 69.5066843151 68.0844667893 67.3082233775 71.5902965807 76.8885781632 72.1654720777 -80.4708943189 49.5021483467 41.6095572691 83.8486865559 46.9624619942 78.6665589963 46.9231 79.8718 50.6583547714 83.1144793358 51.6803004439 89.9970275485 45.8019347617 81.6582677456 46.9095386323 85.6022611586 49.0409566406 77.3625832678 56.5944213157 86.9270062202 50.4441175504 75.0132995059 38.714899408 76.3969778172 43.7446196844 71.2679610751 -71.9641671122 41.8742826668 53.0030253202 84.6747986012 50.5649338128 79.0772985529 48.2051 84.1026 50.2033658114 82.9826568621 53.0965573571 89.540243712 45.835020171 82.7425919919 46.9051031994 81.5656424136 51.8868880222 80.1136076892 56.5313372853 87.4998110713 50.4672293846 90.281464505 35.5480948795 89.3049758407 39.0542484428 83.1190207043 -78.0794214417 93.1333167652 54.7141710591 84.0431280734 53.7196765696 79.306936167 50.0 85.2564 50.9295476993 82.8460411252 42.2422929389 89.7085785936 40.0788064496 80.2471852579 49.8069877356 82.085025592 57.968981435 85.3560457326 59.6521583697 87.8094615921 50.210881979 80.9417717084 36.7106729424 86.0749670003 39.9316746322 79.9391880033 -41.6775957748 30.2012640846 44.2916687157 82.9094412144 54.2002323226 79.3250023354 53.0769 85.2564 50.1786718465 83.1846223323 56.2338756577 89.635308157 51.9829247003 82.2043834675 48.8632418189 83.1583574877 54.855664646 83.7833976656 56.6365087005 85.637495556 50.205406955 90.6405359009 64.9126624396 87.6457960566 39.762826107 76.4059674459 -65.953595185 31.1474060835 49.1917219555 85.8762291224 47.875198343 78.6758534295 55.3846 86.0256 50.422698058 82.9099014671 54.6809865825 89.9645994589 57.337906053 84.9995188696 66.7481899024 84.0048341677 54.1331954613 83.7939960824 58.6722880025 90.077160307 50.4382906648 92.3547602661 66.0641357377 89.2086685815 41.323383287 80.2118685969 -62.9344593731 31.9163906992 53.1013817819 84.5476586881 51.4734585149 79.1598669963 56.6667 86.0256 50.4642248341 82.9353284131 57.9026866088 89.6184017294 62.3362327624 83.2465678988 62.3597662841 90.2436599196 51.3088252568 80.1363911553 58.2216127264 90.4110187715 50.3735188462 82.2192057218 64.9359294757 87.0721381174 41.7873232275 83.9880779935 -64.3737979844 28.8625834061 51.599848153 84.8198214898 52.7512222162 79.2936599344 56.1538 82.9487 50.4492703306 83.9699203811 51.8202625467 89.8341303987 58.5059100905 81.7455863409 47.3748964954 83.4715555182 61.1203943844 88.7040529069 57.9146644768 89.9538027677 50.253683689 81.7438287242 67.0265507063 76.2225153324 39.7556490651 79.6570966361 -72.5093283599 39.5401302526 54.3797219484 84.2403555494 48.5622230047 78.7823835275 53.8462 80.641 49.9283802752 82.9936654888 56.8283953362 89.5836647251 56.2479351377 83.513949493 47.4464667165 82.622124014 55.5638650304 85.8103751441 55.3155090581 80.2518606944 50.2504708592 71.9513852104 65.3140322257 77.9387440331 40.7498793985 77.204657286 -30.0522898741 96.6175423534 46.4807681047 83.518211666 53.0812416908 79.2972507654 51.2821 78.718 50.4880136436 83.0995191232 58.2722785867 69.9002729578 55.4065208295 80.2666719669 53.385280634 78.8711362558 55.044123361 83.5414106324 54.5757285877 77.5362884724 50.0870583609 73.9518736268 64.2952177764 84.1258572133 43.8568984983 77.6098436482 -28.0033242354 46.6721919544 53.174656268 84.2605679876 56.475383374 79.1678400979 50.0 78.718 49.9649053793 83.7083177013 51.8281274487 70.3526492053 53.2839688423 84.2635992213 49.4527077733 79.9306028004 50.4441853247 78.1958883115 54.4130936504 78.2290865878 50.2310718829 80.6473870602 38.7980121423 74.6106304299 42.3678224956 73.6382718294 -75.4012268619 88.6390766207 45.272002941 85.2370794195 49.570911405 79.0381238831 47.9487 77.5641 50.7521082559 82.9019500961 44.8499842618 70.3952029719 44.6767615455 80.2382855483 54.8026825363 75.3639849893 43.8277089981 71.0470522123 55.0745059041 79.8175464166 50.2077411201 73.1145035177 34.6757573144 79.6375987743 39.4629598364 78.459973898 -38.9800154218 87.322160691 36.03340215 53.371687755 37.6277032991 74.5575043418 29.7436 59.8718 27.4224292143 51.4388762325 24.2703823928 50.4855676918 30.0112642472 68.1467563743 37.0882451339 60.857227495 37.7914323802 66.8546410275 29.4329605156 60.8017765444 30.4995061735 53.6125276703 30.7192619427 58.812060082 44.5173339492 70.3693498252 -65.2199135479 84.6829549336 28.2711941653 72.840231257 32.2637000863 70.3166389012 29.7436 62.1795 27.6740833991 51.3041121452 34.7424714403 70.3399348879 29.7452539206 74.4045268347 32.3759619238 68.5108204552 29.292783111 59.79199693 29.4226860665 63.0684648229 30.5203612407 65.1004090686 34.1051509268 64.1291972267 41.6360929963 76.2492229205 -73.0539899616 29.3808085571 25.0548060847 71.5485979221 32.542380336 70.7024172063 31.2821 62.5641 27.5373953151 51.5936540762 37.7993149267 70.4695358763 37.3099483549 72.5861552003 37.4770138847 63.7886855744 32.1920969515 60.2006509147 29.0056141576 63.3907513334 30.4549558873 63.5017787814 33.4233718511 67.0957433613 45.4229321817 69.0955016709 -34.3983616372 59.5444469033 64.7588870014 82.3152236405 52.8250171923 85.5781338795 57.9487 99.4872 52.2633473764 94.2493278274 49.7869410649 89.6580469945 61.1359469093 82.5375693646 52.4066488967 96.4254150057 51.4339355233 99.2568672885 58.4618385916 90.2653263904 50.4164772292 97.2696919618 66.3138096741 98.9310270371 42.0028562742 97.8376147235 -43.4904501336 40.782542065 63.1445274767 85.2366963264 58.5282895407 78.9787042541 61.7949 99.1026 51.7372816586 92.9791175339 59.751497414 89.7319325216 61.7206191907 85.7056934492 50.2970432655 99.6441791736 67.4641398118 95.999963905 57.997804739 92.1599086113 69.5250693831 93.5980502089 65.3096314498 98.1651568396 76.1056290282 95.3049805754 -55.138737967 30.7257603575 50.4246786898 85.1747475891 56.265834263 79.2529394854 64.8718 97.5641 75.8709636929 88.3864417377 64.7747992444 90.132316736 63.6093798555 83.3600288362 61.5633243835 89.8641388605 65.1507052757 92.5026337368 57.5494740761 90.7489065581 69.5202566002 97.1938760167 65.7140760682 99.2837639519 79.2025626772 92.4072156264 -43.6843934333 32.8230098696 70.6449962629 82.4309187567 57.0758924193 79.1939225382 68.4615 94.1026 75.2443262052 83.9034973788 70.8444476004 89.9242090159 70.7237162322 87.1522075165 61.0865603919 89.2243556475 65.5834276962 93.5170609365 59.5299284566 88.3272741451 69.4989700668 92.1204899292 67.5934531877 90.9519098246 84.8482440854 95.4248045304 -35.9036097344 91.1118630801 63.1490490834 83.9468553485 54.2564023808 79.3138648504 70.7692 91.0256 75.1982952889 83.4623033365 70.6059455116 90.042061963 75.0063926418 82.4258351563 59.2364904011 91.4811005766 69.2889731377 99.5795911254 58.2493910631 92.1296814778 69.5286156637 91.6220882409 66.3880173202 89.2812304744 81.5644753207 83.7929072262 -45.3780188805 29.1692166544 62.8240245172 84.9661859507 62.6856712475 78.1747432719 72.0513 86.4103 75.7010415321 82.9194588559 78.6424883907 89.5512032307 75.0925497814 83.4373907092 66.1540050863 84.1119370252 81.2694405511 80.635944933 58.0245140126 91.6944211685 69.4997669273 84.47968329 67.8061881135 91.6881773501 85.4461864009 83.0782938448 -39.7774828713 43.75581895 70.236869515 82.1711510621 68.0856058616 76.1123175001 73.8462 83.3333 75.4793396586 82.8840513891 70.810692254 89.5926540118 70.3279291834 82.0293546968 67.2313347767 85.055640808 78.5757448361 76.8157030942 58.3821244904 90.5534760692 69.5038013697 81.8154056166 70.9626103129 86.5181213517 80.3124913784 76.5162389518 -38.6644611569 33.3172384774 70.0427352449 80.3850213547 65.1181219632 77.3991884543 75.1282 79.1026 75.1945668739 82.9321157846 66.7699466213 70.4233752108 70.8633308324 79.4081747457 73.7099068702 78.2706108976 74.1867734997 74.3442041467 62.5667590405 77.7439347591 69.4875893963 81.8359875567 71.9936371714 78.8209480695 80.7662322969 79.8960656796 -39.0440366877 84.6760108316 72.5706234458 80.9712184346 68.7456312971 75.7900554548 76.6667 75.2564 74.8202539551 82.9623887856 73.3363681612 70.3169271705 75.7799048195 79.8861994199 76.5804513576 72.8915336827 72.622209414 74.5269656511 72.1758243064 63.1289294176 69.5191157281 71.1159672213 72.4798343312 80.8006830564 82.6701124763 81.711479845 -91.6399614428 79.4066030605 75.1307160373 79.9840931403 63.0902251412 78.041409495 77.6923 71.4103 75.1643404933 83.0349971693 73.1051481627 70.300511881 78.0936590733 77.759558184 78.8765811639 71.5625276804 75.1415195918 73.8196684648 79.4727615693 63.4086861199 69.5043762947 70.4934438205 74.0784292286 72.9515308522 77.1636254828 73.5237189623 -47.4881326771 85.3899333808 83.2939057345 70.7784317876 77.2045893204 69.3121497658 79.7436 66.7949 75.2633555474 82.9452793018 74.6782541432 70.4453141903 76.7457612243 71.9733600919 77.4490021363 72.4473094857 75.168667839 73.9004448762 80.3577008812 63.2954375404 69.5125730109 61.497563947 72.0362880189 69.7160484161 74.0707000309 68.4795642895 -44.5902125769 22.0340116412 79.6642622775 73.9323097157 82.0339084758 62.6818697859 81.7949 60.2564 77.7564189331 51.1517703255 80.240250951 70.3543089613 78.4646034437 69.7493008209 86.8537073476 65.5433240194 85.3242028911 55.3885788252 78.7572361375 53.3326200111 89.4942005622 62.2343269676 72.9981862717 59.6656447092 74.1142308687 71.9263635185 -39.0896145478 70.4661940802 88.4321025254 64.6242470023 81.5779459183 63.8949255005 83.3333 55.2564 77.9544350549 50.4755789715 70.9468490729 50.4727017878 76.7456848486 70.0440572588 79.806531301 52.7231797883 84.090507598 56.301968006 82.5402395934 56.5410522935 89.5001240511 53.2163306165 74.8121312339 57.5898837631 68.942075458 62.635150577 -42.2293783752 19.9140684075 89.1155590082 64.0015066388 85.1801432805 53.7816766607 85.1282 51.4103 77.0833377719 52.1577992662 88.6626638086 50.4553260208 85.161682187 65.5124047088 76.9882734745 54.2027848022 86.2562238205 55.8309303793 86.4358971909 59.7927618087 89.490246254 48.890796551 73.0273050243 46.2741878725 73.2420023309 67.3532711288 -37.0003871448 60.264279248 89.0921976305 57.768193047 84.2868249936 57.2561680896 86.4103 47.5641 76.0635502513 52.1046520613 84.405952832 50.4948658866 91.8083525691 34.9655944844 79.4793575017 49.1410154455 92.5487933153 42.9019079116 79.4886844186 53.6516742641 89.49778562 46.6299066746 77.067111934 51.4869181896 71.8670131631 34.3291575887 -39.0520864793 70.6525028457 91.7260057727 52.6233532573 85.6224867126 45.0241658646 87.9487 46.0256 77.6820163247 51.1656378096 75.9871047068 50.4919669557 91.8874485924 34.7692538555 82.3041595613 46.9909036215 90.3426345089 37.7803861004 81.5304203188 56.0253645659 89.5048511306 48.4234081578 75.0171368727 46.6224426489 71.5691888532 34.699218607 -37.4884147432 60.8144048511 91.7355387598 48.9702108877 85.6044990156 50.7623379055 89.4872 42.5641 76.8780819814 51.8675622958 91.1120854191 50.489389393 92.2484016622 32.3771544295 90.700691247 41.4667030063 85.8191629042 37.5713861819 79.1867885665 53.2347918468 89.5010526463 43.3473436951 76.6653077326 38.440250966 71.4829096934 34.6805169926 -69.3595594592 65.5213545959 91.5078881747 53.3126520851 84.4247858276 38.761410164 93.3333 39.8718 76.9485027211 51.9075165439 98.2881232748 30.6039193719 91.782532741 34.3842946521 95.4434878077 36.1897016148 90.4418181137 40.1331134204 77.8990579454 51.8224583343 89.4888412549 40.6250912209 77.9158742329 45.9268433826 78.1285220512 23.139231835 -43.542775926 62.4603112824 88.239001904 31.4774348786 85.570985078 44.5600964397 95.3846 36.7949 77.8640547124 49.6625455341 95.0652748396 30.5004028652 96.0805193672 28.0536005018 94.9974880471 35.0953011066 92.0175590822 39.9179311324 75.1307142075 23.3724419733 89.5016235276 45.8151791904 73.7420460836 39.1209853231 78.789825254 25.5971933466 -39.8112302539 65.3348328092 88.5305192041 30.4760310098 84.6267242006 39.4870775084 98.2051 33.718 75.7714500936 17.1112512057 95.2492339568 30.4594542144 92.2478988278 28.0707514904 91.517847246 36.8902631263 95.2605278366 46.008830272 76.0580137544 16.3837496882 89.497529662 33.901372671 75.3298157891 32.8303519164 77.7815467962 23.4472144745 -70.0689259404 7.59346560899 55.3651603364 30.4458502777 73.3961549252 22.8162479426 56.6667 40.641 52.3315691301 51.8788603463 49.6561262881 50.4313198677 57.662276619 24.0606079328 62.2926421901 40.0295333141 41.2902998033 42.3115524113 57.6146743876 33.8224476502 50.4390920216 37.670771798 63.4104355141 38.377735614 48.5306293681 34.1524500077 -70.0405435824 77.1438066024 62.5602588694 30.4471356661 73.5074477338 22.8569887644 59.2308 38.3333 52.5928183721 51.3915915185 63.0186009279 30.4925819262 59.8469248931 26.1777124879 59.2034756073 41.2706776667 66.4313243161 36.5361097353 56.1713975295 32.1179887719 50.179850128 36.8075593352 68.856486669 43.084147214 65.5307814724 42.2810847015 -39.505789079 74.8516272173 58.0066691227 30.2537212987 63.3564881135 17.7240688424 60.7692 33.718 50.4770477213 17.0482894119 70.5382006004 30.4835582526 64.0670962137 27.7765005568 65.2563927704 32.8541041237 63.9863468858 32.7134316129 66.2878905956 26.1171097453 69.5050706711 33.7401525035 66.3377850519 33.3065100022 70.1180341426 36.4481847905 -62.5168908529 66.4847322418 55.0671179917 29.0115351997 72.9590468722 22.5114998325 63.0769 29.1026 75.2964750909 17.0156531888 70.0868697658 30.5100448173 60.9815013975 23.9342981582 62.4447424979 35.4149992261 61.2696820905 29.3857240482 67.8817196169 24.2360184109 69.5798670452 21.9352367455 64.2037185014 26.6441143003 70.4722626065 36.0880530553 -72.1399254065 0.0151193251552 61.6147759612 29.9943911942 64.0208416469 17.8835812521 64.1026 25.2564 75.573954129 17.0621921415 62.6856904424 30.5464245575 59.9670579092 26.8025832185 72.4624062907 27.6242638254 60.7493891909 28.0517363383 64.0280813016 27.6726855125 69.5064719492 22.7286628748 64.4986348346 22.863501327 76.7911300701 28.0720081685 -45.2515760666 70.0034213192 68.5431435378 35.6578321706 61.124463241 17.1163533592 64.359 24.1026 75.4005271583 17.0411068868 69.239417959 30.5060052447 62.7074541119 21.9955085787 69.9194262552 29.3110927191 64.6552305634 16.6122765086 77.4966517463 14.9485235577 69.4994788728 16.3374699821 68.8909906096 27.2962056672 72.4093545935 33.0392569664 -42.0633045627 2.33519661206 77.7061096487 20.3042601852 75.6742073297 24.7825009657 74.359 22.9487 75.8709908356 17.1348939075 80.4002336791 30.4850063746 69.1952352018 22.709792166 77.7628590782 23.4913735562 68.6382807138 18.5080128963 77.6346517616 14.461853927 69.500264751 27.1360239942 72.371523338 21.9616397473 76.271299815 25.5695441452 -36.3556951539 6.0058486497 68.4530459999 13.0355285908 69.5135831574 20.2793578927 71.2821 22.9487 75.6058847579 17.0677230553 74.6478782284 30.6738628584 65.5923626471 20.6941731019 72.7784101997 26.9005353695 65.7766651568 16.7876560655 77.8637264289 14.6106776491 69.5041565157 23.6689181006 69.7654249704 19.999850454 76.9532591384 27.7706107405 -30.3918276596 42.75961287 68.2572064402 12.3846391542 58.5721354669 16.642703808 67.9487 22.1795 75.8955770466 17.1699497051 65.9952727194 30.4750241915 64.6170985343 22.4431455531 65.198316667 33.0271717859 71.1341263421 19.3859575313 77.3381581659 15.890054658 69.5322795847 21.8509008151 68.6213124429 18.9156764428 75.1077300904 30.0169834157 -36.4490038543 50.5462690659 70.2554746739 13.2503849689 70.9819409865 21.1458881039 65.8974 20.2564 75.7465632023 17.1057165115 67.0885590036 30.4945239508 59.191159132 17.767901807 69.4598588916 30.1970315053 63.7231835039 14.5214211957 76.1804165329 15.9125737458 69.5001114942 11.7916260054 64.2977371716 20.4287496884 75.9692852004 29.0388653753 -40.467576002 60.0275120878 65.0443252778 11.0008414785 65.2230303973 18.3971951223 63.0769 19.1026 75.1423414811 16.7549238932 59.8525770283 10.5431877763 55.8396313724 21.6185696663 59.3150555205 12.3543136666 73.0014940701 24.0738125371 77.2526510945 15.1515170246 69.5048154243 15.6143582625 66.6992650902 18.5910853404 75.2838870131 24.8747531721 -81.7246168002 6.03754484635 60.0922466078 11.8721117093 65.3150439093 18.3337482986 61.2821 19.1026 50.6617795563 17.0781405153 53.7489900744 10.5225745323 59.0007586238 21.1710074885 51.487713355 20.8436106875 72.436657497 20.5783451136 77.4133752817 15.2219279762 69.5034745234 17.1690897689 67.5445276811 16.4479381344 77.0522406385 27.9056390894 -48.8231974964 76.6353305783 52.9920289667 9.90666848335 64.0102406778 17.9244781912 58.7179 18.3333 50.6998506392 17.0851843836 58.2390955164 10.4948990284 56.4593711284 25.1445642082 57.1124475509 15.3217851726 68.4127671225 19.9131265443 76.7318493982 16.2168461441 50.3774019862 16.5060150613 63.9469519836 18.6928454476 42.8267563374 25.5569594065 -35.6205617651 57.2860155789 50.1446289773 12.2115430946 57.1409359294 16.5986105437 55.1282 18.3333 50.9189408654 17.1476047571 54.810300495 10.7249292845 52.3581053558 19.4170541222 50.2244178157 19.1709698793 52.8583150204 20.2350610035 49.4711054118 25.0630193062 50.3394254813 18.7234506861 64.3881920705 15.7728122808 42.0806606852 24.9465264454 -50.5839631148 71.8066161014 46.5086141932 11.2071344881 56.3651532552 16.3265463742 52.3077 18.3333 50.7252585405 16.9074698083 52.9767123211 10.859490276 51.8116637782 17.0334945954 50.4338611907 18.5348708515 53.5299526734 24.2419155845 42.4765399431 18.3384735636 50.0936501016 11.6077134241 65.570045691 23.7657582226 39.8783046498 18.3249292157 -61.8564651063 71.7927431642 43.8070319612 11.3189448865 43.046183241 18.4025593948 49.7436 17.5641 51.2638712274 17.1623497098 52.2038939272 10.4956369212 54.222845621 19.503665544 56.3803054046 14.7884597101 50.7597410528 18.0893946004 43.5951158622 19.9942009773 49.9341271714 22.2561397401 38.4028368753 19.0468586722 36.4754991726 17.7189477884 -39.3237560262 59.3008196656 57.8178546865 10.9451424306 52.0817555582 16.4538745312 47.4359 16.0256 51.2509196468 17.2404558616 50.7451622429 10.5249055723 49.0187553996 23.4892740197 54.8635045999 14.009611915 42.53480844 10.1698684594 50.3399696652 26.4713966117 50.2888186122 13.436410466 37.8323600164 14.4694894463 37.1030736419 15.4456116477 -42.1856791429 66.0348978235 50.9404926595 9.69154713447 42.8790365154 18.4894110252 44.8718 13.718 50.7851572052 17.1801964842 42.7131936166 10.4875472428 35.6535812004 17.5145080595 45.4998576493 3.98542629287 41.5303978512 13.3962907452 40.7489802561 16.1821416622 50.2879944267 12.8722822086 36.9041611715 13.5838157511 34.1489682077 13.5238698677 -30.8469189898 37.3416401041 63.4973230835 11.9140691707 51.5669948027 16.4857136372 48.7179 14.8718 50.5013965822 17.1057707188 45.3172255098 10.4958733944 46.965785757 16.7892016247 49.9367453057 21.5900969196 46.7145179915 15.6132744651 38.3865255803 14.5802151464 50.2347207 20.4044729196 36.286143052 17.1057707266 37.5794219113 15.0158901333 -29.3462004281 42.1487418312 50.0164829541 11.9338520864 58.8391251216 16.6376352154 51.2821 14.8718 50.7336745372 16.9929634144 53.116915041 10.5068605973 49.5842392555 16.7579160147 53.8098729164 17.7664144512 48.4506176492 16.2122782224 38.4040152085 14.4519484496 50.2797016367 15.608737571 62.7866325947 13.9189931024 41.0643057894 12.2144689321 -82.1105579783 1.21055166293 58.6367650809 11.9747210652 60.7557891374 17.0332315064 54.1026 14.8718 50.7113785398 17.0883158538 57.049801823 10.5081369414 51.8562999655 21.969084007 59.8526782804 9.03994992133 59.6326101247 7.05803719901 38.7642788913 14.3655904744 50.2530067222 11.3020842468 66.8176792234 11.4124971575 42.195266147 20.9193721857 -38.3020058088 60.0177857932 54.7302890863 11.4128826732 54.7611561181 16.3661890954 56.1538 14.1026 50.8127448984 16.5727180506 54.8546309465 10.4639151924 55.9528886329 16.7897576048 48.1756567942 19.5212153543 61.6185682473 13.4108227184 41.4701423265 17.2780334445 50.4741411162 12.4619966676 66.755021412 18.0853051378 36.6055782383 15.748875885 -56.5841530218 70.512514809 65.8755478023 11.7324363627 53.5699050003 16.3397125506 52.0513 12.5641 51.014232955 17.2210955293 54.9248425638 10.498331942 51.6637862607 19.9251251977 50.7417190667 17.8384056087 48.8339891275 17.7259558674 47.1554048146 22.3779325323 50.1084399725 18.2590443442 65.4155283864 10.4635122068 39.4311612098 19.3238407541 -33.3393742865 0.5091067352 57.0609827131 9.92056085444 48.9146182416 16.7986370826 48.7179 11.0256 50.3535214052 17.0647430794 63.3095540306 10.4888142392 45.8596693166 17.1289036754 58.4597612975 14.1486628546 45.3426531889 13.9088123935 39.5825667453 17.6484528361 49.919058249 3.7168538855 36.9463314417 13.5143774996 37.5620763361 11.9284457044 -78.7742390407 35.4841012146 46.819907946 10.4946526804 52.6461373069 16.3848363836 47.1795 9.8718 50.4355295722 17.0651685025 49.7494984129 10.4858029097 39.7403795202 18.7410514146 50.1145975193 1.74146171252 42.4150122932 11.6937810961 41.7402438167 17.8293243101 50.0836645085 10.2358819024 37.8254347391 9.60103429534 34.2381100444 13.7131582305 -27.9226442446 25.9868781844 38.3593948725 13.431322624 49.8384929905 16.6763302777 46.1538 6.0256 50.6309819633 17.0765223454 41.1693104232 10.4973002433 52.8112216438 17.8549881176 45.4203820755 5.37240899542 58.9347650647 9.36920121911 39.3118718889 15.6407169743 50.3522608355 12.4686005273 36.722837943 9.33330210686 34.1409410386 4.57766135024 -71.6978651182 10.8681445111 47.3154157773 12.853451784 52.6840664009 16.4079627826 50.5128 9.4872 51.0668072036 17.2088597095 46.6883404976 10.5075321353 56.7891043639 17.3061155207 51.3532681954 0.30387242055 60.6836260407 10.6326415569 41.6798476925 17.7459190133 50.4975722411 7.40962523419 67.0733217806 6.04921458162 36.6592676972 17.6819644553 -74.1383313856 49.1739189791 55.0519165396 11.9499886181 65.8072190346 18.1407649473 53.8462 10.2564 50.7923547263 17.2042143387 52.4189219215 10.4959063596 54.1145317806 17.6735343896 53.091704424 19.2900304453 54.8117983683 6.56964870325 39.0874644519 15.1223039378 50.0163058346 16.2536847911 64.601818107 12.0019169643 40.6437192795 20.2408613716 -32.579020066 1.80811559665 50.5159602579 9.76559162187 60.6739209233 17.0518023644 57.4359 10.2564 50.5512780561 17.0846551833 50.8784883796 10.5030898746 55.571119732 17.369574581 58.9431665444 11.6128777918 61.5868424253 11.0275858356 41.4815028637 18.0474374446 50.4682212165 13.3567024555 65.4372821806 15.5453860785 36.7924815919 13.3921347304 -59.83218542 69.1525081443 49.6774146514 10.3831325083 63.394712866 17.7005351354 60.0 10.641 50.5597580562 17.0938837744 57.0533061394 10.5140692392 65.3064624975 18.1973548277 59.3109904219 10.5721363739 59.7160077628 11.5190968617 77.6060865492 15.1628725383 69.5506789842 9.03298756904 67.0040223792 15.3458266393 75.9483006427 22.7447959146 -35.0306285457 12.5366493416 67.2806595188 14.1286515323 64.8020094467 18.2168786335 64.1026 10.641 75.3259785471 15.7718919896 70.7864511597 10.5136950409 66.9907491546 17.2228706509 68.2221986595 1.13388036584 58.4163840114 5.64577748328 75.9826615205 16.3069223786 69.5186727475 15.1804864298 66.7241920702 5.24980548075 76.8406924717 20.9185472197 -74.3001198284 42.4770945921 66.1730182606 12.0379170217 65.0259398087 18.2878245002 66.9231 10.641 75.04472578 17.0042622562 56.7721495656 10.5032723508 72.1567189391 16.8142747914 62.5117719862 7.49123248279 59.243809528 10.5401420943 76.9457572424 15.858478334 69.5034574098 16.8783132661 68.3076187632 13.2809165227 81.9504776395 19.8745582085 -63.2501970628 65.9524861966 61.0885441374 10.0845344144 65.7555515404 18.5162158303 71.2821 10.641 75.2870877232 16.1749349097 75.4772220231 10.5185680355 70.9805804021 17.15575228 69.3800563617 1.48813233299 56.5943646385 8.91734404523 77.5437200743 15.253949149 69.4994868291 10.2078266798 68.7680482759 13.5214565521 81.6437056853 16.5917599845 -34.1730737648 25.6936743092 66.0530857731 13.3802260073 69.9677341245 20.5389887758 74.359 10.641 75.2399699777 17.0318474866 76.7659121498 10.496999064 72.2220822073 19.3692816955 82.8002478704 18.5293762368 68.4925132247 17.9873634289 77.5847398447 15.8300393854 69.5275296829 9.99229666551 74.1672717461 5.34988087254 84.6102197758 14.2477619017 -40.9138319319 38.5590195509 72.6699892655 15.2342259409 68.8927840241 20.0324488429 78.2051 10.641 75.1524592023 17.0049424015 77.621049129 10.4958742318 79.7001583481 20.740218365 85.1603321777 15.0183695439 65.0059431717 15.9126686464 76.8223042562 15.5951653195 69.5190952617 11.4226857018 64.9003579699 16.2452583689 84.1028794336 8.98065579006 -62.8332930874 62.5108942269 61.503472503 10.8284144754 61.8265465579 17.2696554452 67.9487 8.718 75.9618400861 16.6948422289 65.4144870523 10.6818401251 64.2491606461 14.9139624607 67.095847697 0.601490941965 63.9210367078 15.6209610332 77.3485716647 15.7745292395 69.5618994268 15.9998037858 68.7634335463 8.70057294385 80.18002545 19.0888441297 -42.4767923803 56.7312899691 68.995028632 13.9943105293 60.7879700808 17.0462048583 68.4615 5.2564 75.4480625064 17.0451494125 69.9647805002 10.5072329326 66.9432914334 20.1891325034 62.307750573 5.74997471945 53.5129705438 5.87510952836 77.5731526867 14.7806458332 69.5010445197 5.95583353945 66.8169139163 12.2732943361 80.5732729943 8.36563890168 -52.0334562787 64.5666620298 78.2499161658 17.8832409068 61.9156096372 17.2821912228 68.2051 2.9487 75.7593838168 16.9429296545 64.9697382623 10.4821595117 66.9416766634 18.2435093914 59.9704150458 7.9133104735 57.8328062743 9.22238462333 77.9726106776 14.9556987461 69.524635976 2.73476016988 67.3093466467 0.217006270014 79.0043268649 10.6281977654 -48.9070429644 74.2877488252 36.4819805699 15.1627600944 45.0289975331 17.7244473049 37.6923 25.7692 50.378262298 16.9462798113 38.0502407785 30.4079460343 39.6482666468 22.7313663718 39.9116351739 32.1440226145 31.0329174395 28.9281418896 41.5289197586 24.9164251896 30.4995937106 26.0855615691 34.7318293093 19.6017950724 40.0481864651 24.2614879334 -59.8518383854 72.9583909677 50.9677483784 29.6797742875 39.9213628084 19.7018495281 39.4872 25.3846 50.5336350094 17.0195813691 42.2241202189 30.4980371582 37.9497836858 26.5217492344 39.5965009783 32.1361317349 31.3332440932 32.5385694785 43.7225508034 19.0773278036 30.487391709 19.7794703733 33.6744421497 26.09049021 34.7945935378 13.9696834611 -48.9604602016 72.6295257275 91.1910536059 46.6743428399 84.7942778751 55.5686503735 91.2821 41.5385 77.5009073165 50.1669859475 87.6984642949 50.4899365296 86.5043905611 34.8940252874 95.5934163965 33.2341293953 86.4015498466 38.7469331743 79.3260781751 52.9003912897 89.5001804719 31.9789174357 75.6272550236 37.128751949 79.221764441 22.0945914837 -46.8448551673 36.7917136918 55.8637684939 85.3364867573 55.6629587449 83.3564796243 50.0 95.7692 50.6911241884 87.5139604241 52.3603310701 89.7043170068 50.8154853559 78.950838805 52.772288113 99.6134716833 47.4421116354 98.1843018589 56.6639740815 87.9401250067 50.4102716314 98.6283694405 40.6101254225 89.1362398699 36.0308797708 93.1217332374 -39.9630215796 42.9449148487 49.2805947987 84.0488228306 50.4922475132 78.997532065 47.9487 95.0 49.9903949511 83.997356924 51.4067867898 89.7121385499 51.1834666638 85.1288546497 53.1786144535 98.6038611855 46.2647413368 94.1161921613 57.8217892319 90.6931665489 50.3259243927 94.9946308528 39.1143664549 96.4817513202 34.4995583102 86.6099850511 -66.704943997 32.0150954299 43.3685015447 84.3321772039 51.4671011416 79.2018445773 44.1026 92.6923 50.1271820272 82.990749996 43.588468468 89.7455365536 40.8638552293 82.9261452983 37.6270534612 93.060672952 40.1638157437 87.4486723469 58.2431718991 92.1043278652 50.1040315041 95.0885380179 34.5838289299 89.5889019877 31.1068665551 89.4616352367 diff --git a/example/neural_spline_flow/data/data_proc.jl b/example/neural_spline_flow/data/data_proc.jl deleted file mode 100644 index 8d572ca1..00000000 --- a/example/neural_spline_flow/data/data_proc.jl +++ /dev/null @@ -1,7 +0,0 @@ -using CSV, DataFrames - -# Read the .tsv file -dat = CSV.read("example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv", DataFrame) -dat_dino = permutedims(hcat(dat.dino, dat.dino_1)[2:end, :]) -dat_dino = map(x -> parse(Float32, x), dat_dino) -dat = vcat(dat_dino, dat_dino) From 00387d3853a761efe80bea7208cc56b6d31e8734 Mon Sep 17 00:00:00 2001 From: Zuheng Date: Thu, 17 Aug 2023 02:01:35 -0700 Subject: [PATCH 39/40] rm @view to avoid zygote mutation error --- .../data/DatasaurusDozen-Wide.tsv | 144 ++++++++++++++++++ example/neural_spline_flow/data/data_proc.jl | 7 + src/objectives/loglikelihood.jl | 4 +- 3 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv create mode 100644 example/neural_spline_flow/data/data_proc.jl diff --git a/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv new file mode 100644 index 00000000..7f8dda94 --- /dev/null +++ b/example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv @@ -0,0 +1,144 @@ +away away bullseye bullseye circle circle dino dino dots dots h_lines h_lines high_lines high_lines slant_down slant_down slant_up slant_up star star v_lines v_lines wide_lines wide_lines x_shape x_shape +x y x y x y x y x y x y x y x y x y x y x y x y x y +32.3311102266 61.411101248 51.2038911373 83.3397766053 55.9930301513 79.2772636977 55.3846 97.1795 51.1479167122 90.8674123313 53.366566866 90.2080300059 57.6132335477 83.9051707998 52.8720214902 97.3432226575 47.6952008891 95.2411869282 58.2136082599 91.881891513 50.4815081703 93.2227013657 65.8155400946 95.5883740582 38.3377571839 92.472719051 +53.4214628807 26.1868803879 58.9744699018 85.499817612 50.0322537946 79.013071199 51.5385 96.0256 50.5171258092 89.1023945866 52.8019793617 90.0880645063 51.2743918205 82.8179829379 59.0141444945 93.5748748733 44.6099759113 93.075835032 58.1960536923 92.2149886482 50.2824056687 97.6099835723 65.6722651618 91.9334018119 35.7518707905 94.1167680276 +63.92020226 30.8321939163 51.8720726696 85.8297376348 51.2884586593 82.4359398425 46.1538 94.4872 50.2074801993 85.4600473951 47.0541298828 90.458936026 50.7538984114 76.7541289533 56.3751090389 96.30514763 43.8563814822 94.0858717713 58.7182307185 90.310532087 50.1867033389 99.6946801425 39.002716451 92.26183816 32.7672179591 88.5182945794 +70.2895057187 82.5336485877 48.1799307865 85.0451167372 51.1705368873 79.1652941091 42.8205 91.4103 50.0694819181 83.0576695262 42.4484337816 89.5077001153 37.0211840116 81.954469997 37.8391996844 94.359443503 41.5789293984 90.3035665387 57.27837287 89.9076067166 50.3269108629 90.0220534916 37.7953028026 93.5324553607 33.7296067755 88.622265789 +34.1188302357 45.7345513203 41.6832004024 84.0179406019 44.3779145287 78.1646280036 40.7692 88.3333 50.5628463386 82.9378217764 42.7040363241 90.4426288607 42.8817550897 80.1847700514 39.8853734176 90.6346620817 49.1774185559 96.6105324538 58.082020494 92.0081450101 50.4562073315 89.9874101286 35.5139005857 89.5991901138 37.238249327 83.7249284144 +67.6707164012 37.110947969 37.8904155015 82.5674929978 45.0102700721 77.8808631153 38.7179 84.8718 50.288527801 82.9752535659 32.3789386326 90.144142631 37.1557104777 84.9541114129 44.0774051326 84.1258533023 42.6522509813 90.5606397307 57.4894477748 88.0852855629 30.4648469308 82.0892320845 39.2194529062 83.5434818364 36.0271982243 82.0407806505 +53.2591294055 97.4757710964 39.5489736947 80.812601774 48.559816719 78.7883706035 35.641 79.8718 25.5834750822 82.9148911324 32.5307027362 70.1573965085 38.7318581261 83.7893564281 31.4970254888 67.9125621568 30.1233318217 81.1442953304 28.088741319 63.5107944263 30.5015964737 82.3071263462 31.5882013862 77.0280524208 39.2392807768 79.2637238398 +63.5149808671 25.1000785788 39.6495738769 82.6645338713 42.1422682922 76.8806345544 33.0769 77.5641 25.4835833896 82.929084979 33.3652644305 70.4593354392 31.0005254108 82.5794805962 25.9525965456 73.0331803541 31.915519986 79.7904134869 28.0854682136 63.5901969481 30.489569227 77.7298801366 32.6577782482 80.1138612983 39.7845249391 82.2605658962 +67.9805388133 80.9571652197 34.7505970472 80.0110909897 41.0269715735 76.4095884613 28.9744 74.4872 25.4435257049 82.8742004973 32.6560991775 70.0510704992 25.9854996226 74.3493875478 36.4347239148 62.2940454639 27.8877193574 75.4455759934 28.087273053 63.1232828116 30.45024833 79.8348944011 31.6415992679 77.6108331874 35.1660293896 84.1564919839 +67.3724659005 29.720400203 27.560835291 72.8478255927 34.5753090115 72.7248411845 26.1538 71.4103 25.5651134237 82.9240977704 22.9550932697 70.4268951523 23.7145662909 75.0980490792 24.997482691 75.4415616626 27.5482344526 75.1647590544 27.5780252176 62.8210386566 30.5183947985 68.2525829388 30.9871426099 63.9440280336 40.622115916 78.5421042089 +15.5607495229 80.0656402858 24.6355399754 71.6107148256 31.1686007236 69.2454213193 23.0769 66.4103 25.9288442731 82.8211841087 27.1498436318 70.2108058428 23.077319507 71.7205396747 32.2462751352 70.1105868383 20.2097781637 67.5166420988 27.7799191139 63.518147517 30.5116612138 66.0937181831 27.4396322054 65.7444629564 39.181907086 79.8190371976 +71.7907676942 71.0654666627 20.9594648125 66.0414983826 32.6442540714 70.7325550334 22.3077 61.7949 27.5514782612 51.4873865306 26.1669092779 70.5003151212 24.9391368744 72.5802858084 18.1094722949 53.7081103324 24.4434528292 54.4736551228 28.5889998149 63.0240805653 30.4980777147 62.3849850338 30.5783699449 65.5219454667 42.4308889944 75.1336346406 +70.2425464362 24.1095975542 20.6891490538 62.7213052064 26.6666456539 62.9809712052 22.3077 57.1795 27.5304663693 51.4148465629 26.3314570795 50.494556525 17.8934987136 71.1096873431 21.7960992263 49.4995295248 24.6308301331 53.3957388998 28.7391414969 62.7208638859 30.6202277519 60.8600821544 28.809401637 59.6210044666 43.0812558617 75.3467016413 +64.9374355444 81.5542049945 19.2882047369 62.0630593582 26.4592623274 62.6346733709 23.3333 52.9487 27.0955703605 52.0767994399 22.0037091415 50.4994401238 23.6730818572 64.8708391385 24.150491069 47.1885335266 21.0379688969 49.829299645 27.0246032407 62.9018588649 30.4921854231 55.8789168923 27.9395185786 50.9850621313 44.3607194344 70.4742070546 +62.2135245453 21.4758389969 20.0245005746 61.3426238695 25.5380076465 60.6302361093 25.8974 51.0256 27.4392496133 51.7120790527 23.2847479782 50.4743528321 23.7490719056 63.557171895 28.6031956015 44.2237227 27.106242932 54.7195531488 28.8013366963 63.389040388 30.474634333 52.3221596388 33.8499497887 46.2844744465 44.3247788435 71.0459044019 +67.2694004772 18.7089683725 35.4695229971 43.1158849517 26.3207111933 62.3962811186 29.4872 51.0256 27.8782642594 50.70890793 27.6340383923 50.5036668574 32.2151786343 65.6009580191 20.9319996761 51.6462354868 28.0896176553 60.512212549 27.186463838 63.5587296478 30.500998079 48.0696996767 32.628463049 54.0637204929 47.552160202 66.4604037217 +40.5701970446 79.3729634752 36.8943276507 47.7065559687 26.1633874648 62.0268148985 32.8205 51.0256 27.3388689181 51.6530467544 32.0371080007 50.5000661229 29.4368412489 67.5545281141 35.4355325251 34.9610381566 31.1357349152 61.4319127116 29.2851466002 63.3836058254 30.4944213623 45.7734755346 32.4386500903 49.9030194916 48.6649246215 64.9610049438 +74.7411813341 21.1016372041 39.0555497751 55.5469737131 25.2137904611 60.2733986018 35.3846 51.4103 27.6784029741 51.1819891691 29.3365106528 50.461893969 40.0500951983 71.8890335708 44.3773828236 57.6847369215 37.700837521 68.4184052804 39.4029453026 51.1508571967 30.4995445502 49.3820997506 35.8382920833 53.3514883396 46.280631958 68.0284231525 +71.7683189223 20.0110618423 46.9570801494 65.2404073935 26.8595295604 63.1418346727 40.2564 51.4103 52.6356576791 51.4185522584 40.816953133 50.4877601957 37.5747950188 69.9887444412 46.313692406 55.6855917703 43.3926884016 43.5975940692 28.8113284396 61.3578540576 50.144392979 47.0142836417 37.1420429198 49.7419879519 50.5632444849 62.6238279659 +76.1669198143 75.9361704048 37.3104527352 45.2587508999 31.7606691716 69.7976553936 44.1026 52.9487 52.0252141055 52.1230110467 42.7390751173 50.483155992 49.3104686263 67.1687603206 46.3860739231 51.0021645381 51.2623974761 49.6665051258 34.3039579069 56.5421259093 49.9128985877 55.5878389121 34.6445467746 54.4441077764 52.6309642318 59.9639357915 +65.6236574431 15.5828033531 40.0096720019 60.9865825077 39.7932905744 75.8079642168 46.6667 54.1026 52.8811647904 50.6215547554 51.8678937691 50.4932732621 47.3480185524 72.5916115408 42.4954400876 56.9036815315 43.2867129203 43.0568603208 29.6027609801 60.157346722 50.3374274859 57.9553142818 35.675404696 49.6256433471 54.6362064446 57.4938424367 +50.8506336394 13.9876016304 48.0143866846 65.7128195934 45.6174391854 78.1199702277 50.0 55.2564 52.9526073133 50.0747390118 43.3710018924 50.5008373234 50.157414434 71.1617258246 44.3005150328 58.1329592781 55.9059162338 54.2526725342 49.1161568603 63.6600006211 50.4189791992 60.5249117346 64.99458547 49.4621728312 51.346027139 61.6005530364 +33.0240700249 24.4678303872 53.7037759268 66.3894820363 53.7714606827 79.3029316435 53.0769 55.641 52.5205524936 51.5024421035 53.0427983006 50.4875038691 58.2100082697 68.0445380719 46.9882833458 55.4206367313 50.198958949 52.2250295948 39.6175458263 62.9251879574 50.2052589714 57.9242171676 66.1291498342 61.6437178853 57.134855354 53.8779658045 +39.7063261674 84.2752871038 63.0674998921 64.0350004581 56.4514336733 79.2471070208 56.6667 56.0256 52.3428220582 51.8619520908 56.6189102967 50.5033740178 56.6089076561 70.4089460888 52.4215231623 47.867005488 55.761587372 54.0305172396 43.23308466 63.1652187223 50.4353151633 56.4393224734 65.790651882 55.9453737171 54.1712415737 58.0598078952 +45.5964849542 9.76334884943 62.0480325076 63.8458632494 66.0937201989 77.0178182606 59.2308 57.9487 51.9275902141 52.2577906065 70.568289819 50.4962361243 52.8338205303 72.5518862725 64.090998189 68.6509850107 58.3231396505 60.0310591646 64.8927879422 65.8141767574 69.5057889042 59.2849248279 67.5017806138 60.8502274659 50.9423823071 62.0973939351 +42.9680469104 17.9454583961 59.8399667124 64.4767644437 56.9258394194 79.2063186226 61.2821 62.1795 52.7137744894 51.1979443188 67.870887292 70.0680261345 61.8185854576 76.1128240228 63.0168717563 68.2252589114 70.171457528 70.2822212385 62.4901493154 74.5842896072 69.5340709833 64.8929773346 67.6295142176 61.4291060701 66.512924456 59.0712818478 +52.4944067819 16.0511142003 55.1609418175 65.2373081694 58.9882538467 78.9172589554 61.5385 66.4103 50.4438027888 82.9418288173 58.8215465891 70.4557817483 61.7276334143 77.0692384058 67.3025779226 64.4134627173 60.600276471 59.7533088802 68.9880844294 63.232147305 69.5101069302 61.3296244282 63.9100640599 62.9723349305 64.3979095023 56.1748754008 +46.0822757831 23.1104578154 61.2797865792 65.766402504 57.9022660533 79.0808678187 61.7949 69.1026 50.2166950326 83.7523429676 60.1869278445 70.4835512458 62.1922520815 76.1568192103 67.2259568904 62.4764105301 63.7602750188 61.1856411696 62.1056186306 75.9908707599 69.5263228876 56.9952791254 66.5493579687 67.7255886599 68.686593294 62.2373439902 +74.2477082092 20.314187812 60.8349175348 64.6037697063 64.8128669568 77.4869389235 57.4359 55.2564 52.1841801076 51.9752506714 59.4355174849 50.4206326073 58.1187658587 73.1189614684 50.309988715 47.2678818553 54.3894952619 53.811901225 32.4618467439 62.8819029189 49.9472399619 48.2295767261 64.0515417112 48.8200939835 65.0277937057 57.0962573525 +64.5682641863 83.6396338956 61.5205906458 64.7918550418 60.3497935166 78.7294458673 54.8718 49.8718 52.7920973459 51.0733956509 49.5858434795 50.4349813825 58.2704123843 35.5968950425 45.4126389249 53.8707976895 55.2412969947 52.5935746267 41.3272006535 49.0702512739 50.0631041291 58.085037556 68.1932895833 46.5435408096 53.3045031712 40.883844549 +74.0216939058 76.1282745076 36.916543863 41.0952474383 48.890555756 16.8065435154 52.5641 46.0256 52.5897198565 51.338090196 41.7797417848 50.4706422808 59.7665303129 32.6212516009 54.0206071815 47.2698972057 49.0701267502 48.8969446811 44.0071499323 46.4496737846 50.489902373 39.2054119682 65.2783891952 44.8203529292 52.9451780881 40.3846256942 +62.3911805626 5.62307076073 38.5021996714 41.5671556174 47.549506348 17.0714045245 48.2051 38.3333 52.0288486671 52.1768375002 46.1276863394 30.4494107792 52.5355347207 27.1741229419 49.8420222483 50.5765522936 46.0181190216 46.2577233718 44.0740606888 34.5532038906 49.6888638379 42.018514414 34.9976613883 28.6329939351 45.225738199 30.5552259226 +74.189036683 68.1335832223 48.6643707344 30.6806668519 30.8894308257 26.3290372348 49.4872 42.1795 52.729246581 51.2017650475 38.0337837705 50.4908995189 50.8925215809 29.1452839382 50.7132995657 48.7722867825 44.7595874356 41.9746066134 44.0013152386 33.9042073544 50.3334811097 43.588671479 33.9237661692 53.5586885227 50.8433046202 37.7446280981 +28.2367819396 56.1395964513 50.2852524042 30.3379221051 28.972835724 29.0504689185 51.0256 44.1026 52.8843132864 50.4414354501 44.5468412844 50.4159133758 49.9675511775 28.4602033343 55.0455549106 45.9862727459 51.6006241332 50.0324668853 45.0063004454 38.2990195507 50.4467514196 57.2305889423 66.0044543688 52.6846691452 49.6592939398 36.273442735 +75.7719387944 69.8292300322 42.276332666 34.5276361236 35.1613607523 22.7868496473 45.3846 36.4103 52.5093008862 51.4162051508 46.2381175975 30.4956129408 51.3478832298 27.3642602259 37.5437012597 32.8840244657 36.4658820182 35.5241876075 44.4438406096 36.0190833012 50.4209091963 43.3479568967 37.0232562464 32.1184816794 47.1427889348 32.0250641734 +75.8552294691 62.5170442862 54.0317756204 29.6723483149 45.8703691685 17.4893933655 42.8205 32.5641 50.8626843321 17.1456310914 49.7397453317 30.4727248868 37.5551162883 24.5376638386 39.7958978066 28.4432191156 33.3947987327 35.0168013145 42.1787133986 26.492119482 50.3740520508 32.9182406167 34.6982700852 36.4753541349 41.2700565259 21.7218171465 +65.9708570175 72.7448559954 37.3293552552 39.6020423074 32.8314206076 24.7975381459 38.7179 31.4103 50.8914922458 17.1413237277 39.1102518936 30.4786951789 39.1995941778 23.6804956588 28.5233910215 39.7966662481 37.5800715749 38.8315722007 44.0445656189 35.6622382764 30.4768299051 39.4246055913 34.8634441715 30.9392745218 47.1301271555 31.3099029222 +21.7780404779 6.61662530728 41.389522551 37.2960562328 39.4855747899 20.0330500489 35.1282 30.2564 25.8551275976 17.0819086886 39.1428377913 30.7928559394 36.3827221955 25.2380052325 31.3996460012 39.2652632229 36.619722059 36.8114141094 41.6404540171 27.0930954213 30.5034743497 38.3485561381 31.0900650789 32.3837547196 40.9159715783 23.5415923929 +67.7597962473 72.4212015285 40.0746666572 34.6236852027 33.4461829314 24.1906434075 32.5641 32.1795 26.0256445518 16.9250135319 34.6254702671 30.4997410297 41.1006943822 26.5526132161 39.2415652162 33.3614166846 36.1634664356 33.819314199 41.9383300069 24.9915229793 30.5048349283 28.7714879939 34.9213061465 35.4184380529 38.6601639737 21.6895496369 +78.6171953363 52.5752573142 35.3496806211 47.1410731332 21.8635812756 46.0525933457 30.0 36.7949 27.8931727187 50.6619634101 34.7542074707 30.4958650002 28.2178118248 28.2802924847 27.7140565712 42.2500516253 33.1677570349 31.2477411672 44.0539275103 33.5563924949 30.4622184647 32.7504720275 30.343361183 32.5245631742 42.6125508262 24.195053774 +68.5077081898 15.4569189652 34.7637004189 47.6247999245 25.1658708449 35.7666139539 33.5897 41.4103 27.6399679354 51.3990974838 36.7537002386 50.5293043035 33.7970720433 28.6997507433 32.9607035213 35.8231112903 42.6449829318 40.1543216386 39.2067193308 51.5337156971 30.491302622 43.5613740753 35.4833288685 44.4916349303 41.2090395008 24.1287091891 +74.8850211598 25.4166063231 37.0266294472 44.4622930512 22.9594789762 51.2877067934 36.6667 45.641 27.892658895 50.7952815163 37.0193512929 50.4926666972 43.9193873055 27.7978445882 35.4047362397 31.9391621952 47.2250252695 44.5756466185 28.7044492315 61.7775254006 30.449653838 44.9951230008 33.2349104271 38.502483721 50.6492211499 37.4837907435 +66.4549036599 19.8366286542 36.4555695327 40.791843032 25.5860552415 60.8465833585 38.2051 49.1026 52.7977329399 50.6860370901 41.4171152753 50.4697734181 44.4164011084 28.9476923345 51.6422264225 50.8383406788 48.3923630032 50.3315285645 31.7086628996 58.837754374 30.4859942157 47.3479464653 35.4399436182 47.7975176939 52.1319019406 39.3646020815 +77.3178020985 48.3983464352 35.5376642131 48.7293868661 26.3143100672 33.2971681038 29.7436 36.0256 27.5806388053 51.5247612607 22.8171074784 30.4360403387 28.0376027706 26.8038243613 34.207620843 33.5938054385 27.2451932001 29.8195074488 42.8117114739 30.0204484197 30.5020767935 36.4412129198 33.8735095227 37.056028453 39.7972712705 16.083549649 +58.9124603193 75.6677562173 20.4089478892 32.2030304205 33.037301251 24.5644624921 29.7436 32.1795 26.4913985349 17.4053909738 34.800701463 30.4979313851 31.6895761296 26.6664002089 36.5234892198 35.6710483891 30.4043143388 29.0795399241 43.300614891 31.5264261979 30.4780588886 28.020580489 28.5641377966 34.6263997171 43.5498814807 24.3072109395 +57.617447817 8.19480060319 23.4957104659 25.3224681542 36.4688894772 21.8761144725 30.0 29.1026 25.9853178159 17.203722132 27.8918897953 30.4849702595 27.2796128523 19.2572809685 28.4192021281 21.3157482481 30.0807992865 27.5880255173 40.3986329069 16.3470083822 30.5011608787 38.3141385582 31.3195486706 31.6639905316 36.7146633371 17.3818098572 +76.0882257967 59.6799300235 29.5575433634 21.3647774591 26.5135531009 32.9187498178 32.0513 26.7949 26.2014192753 17.0938239128 32.8706391587 30.5023417488 31.2218783581 26.4408557812 26.1604969417 23.0422314726 33.9806339675 35.0330581489 40.4356915763 20.2326706762 30.5047730174 26.4166256707 30.796174379 23.5190343602 37.2246101278 17.9800644828 +57.4660505497 1.50441817488 33.0082339121 15.9850714584 36.0270912125 22.1954385395 35.8974 25.2564 25.8575694746 17.113842659 37.7940494081 30.4926383364 33.5458051034 20.0807081014 40.435327397 27.6400711421 34.2506117649 34.4415865654 40.9365466658 16.9130048364 30.509115326 22.8255471196 34.2333554347 22.4959116015 40.5124478537 22.9622145725 +79.4283834934 45.2107942872 53.9803991821 29.3509867092 42.6810404129 18.5425361977 41.0256 25.2564 50.7046843629 17.0237445436 35.6673163042 30.522035054 39.7363306065 23.4948069772 44.544955314 27.3622426221 51.1411835714 20.6183237784 39.6615736653 15.609355577 50.1156321728 16.6940065479 36.6307877065 31.4768312621 40.2220627884 20.6146217044 +76.3565221496 10.4182411281 52.2343086004 29.7116729865 45.0410771273 17.7345177303 44.1026 25.641 50.8119753465 17.1149252621 48.4422995026 30.5017227753 42.8418426525 20.3339004396 40.1237272008 30.0731619402 49.899592066 19.5640835356 40.899259175 20.7985289466 50.4483013392 21.6402564599 34.5486191018 26.1110878654 44.1249612057 23.3662452858 +64.4050752632 78.5841760758 59.5030766063 30.6696739448 47.3524164557 17.1347454264 47.1795 28.718 50.5648455577 17.0777773216 58.6693732631 30.5068378948 47.0937298879 26.9934706312 42.1848064503 28.5736747331 55.5898773424 26.2992451143 41.9686168283 26.4970725985 50.17317743 29.177864393 37.5196356 29.2676058936 42.3462616039 24.5583269437 +40.6350418091 73.3947503698 41.1637810689 34.3157582514 53.8461940322 16.3873180961 49.4872 31.4103 50.9393039127 16.981021883 47.9169554375 30.4867257545 51.4536583539 26.0229417521 40.0026592892 27.4875100027 55.4820563581 22.8274662445 40.3834058238 21.3912255247 49.9227419194 38.4404729124 62.4256973267 34.0442062707 44.1890843708 28.772625473 +43.9498645857 75.9587156671 48.9930401201 32.0303588363 55.1296393835 16.3532468711 51.5385 34.8718 50.4588548392 17.0385789686 51.8941663939 30.494755684 49.3677068167 22.005580703 57.1252310509 42.7390191388 63.5729393821 32.9386176053 56.5381264538 32.4442454697 50.3112993746 26.8604882737 64.320383794 31.7856294846 45.8142453101 31.056979672 +30.9962205791 71.694404938 59.2692803248 29.6407017654 52.1881199617 16.4182174652 53.5897 37.5641 52.9013640729 50.6905627231 51.7490891643 30.4729933399 48.2534809736 29.2708555331 60.375744968 41.3564579961 59.0250770848 28.3220858801 52.9706912828 29.0401966941 50.3994096114 38.9638389234 65.4455325117 43.2734991936 46.6431192691 31.4072190618 +68.2307689907 80.8725016628 45.4691770035 33.8311927323 58.6726630845 16.6633724322 55.1282 40.641 52.6849534438 51.2944692191 58.2465530013 50.4428744949 61.0029597643 30.0153144324 55.1058961252 46.1490510561 62.4237785929 33.4865952868 54.6209525885 30.3445244537 50.3373626448 40.913386144 64.3256315887 41.6159763773 52.1898883047 39.3677964938 +72.0463894612 12.9180067349 62.6912665406 30.2903738312 82.1443116037 33.1952218669 56.6667 42.1795 52.500088943 51.594356165 57.3106968674 50.4825132895 60.8065608049 27.8683134861 61.7372710245 38.5848440052 47.849907703 48.556624645 65.0990443941 27.2415575588 50.2023180965 47.2129480045 65.264338195 35.7175447772 51.1003173587 38.095699984 +46.5927679682 84.9723827774 73.4286708654 48.5778551303 75.6865711383 24.7415103908 59.2308 44.4872 51.8356372562 52.3357655339 67.9680481127 50.4521589097 59.0674270249 26.7882840217 55.7412443246 46.6535769763 52.2685616825 52.8052401961 63.0559909076 29.7090956748 50.2749326091 49.6234993439 66.388704175 38.8447443366 65.4008341477 42.6140320392 +49.2572183396 81.8814032306 70.846426107 52.282253331 85.344964503 52.886419866 62.3077 46.0256 76.9954121013 52.0455286534 65.9493192681 50.494240794 61.4146337175 31.0393750583 58.2964084158 41.5831977346 68.8510215758 39.573434273 70.9601362323 41.2595012867 69.4943803061 52.6124107059 68.1914228192 45.1694864796 64.8520398267 43.1337348404 +42.7817612539 12.9911884302 71.5390198495 45.5218061588 83.8446535127 58.6160602305 64.8718 46.7949 77.310600475 51.7467300374 55.7470928102 50.4789748003 68.9343694692 31.6198930866 57.0425482213 42.418757739 65.6763960251 35.1093154301 69.8958192404 43.4537592655 69.5193397556 47.384533707 68.5406624515 48.4385263389 63.4038060118 44.847493688 +65.475952195 14.2745856444 67.6208658884 38.0065584658 85.66476066 45.5427527692 67.9487 47.9487 77.9260443352 50.3186604237 68.4030100436 50.4986272318 63.0652438592 34.2357465852 76.7756112073 55.2545987009 77.7468851148 47.3718241139 70.5958928563 41.9647438672 69.5291826079 57.8412666804 67.6614312657 51.0685845873 65.3715488709 42.6959713164 +71.9650826544 17.7102359443 72.4709525633 51.1221348241 77.931004934 68.6900618808 70.5128 53.718 77.2543883433 51.4618248152 76.696465628 50.4938305609 72.2725228706 67.6444437851 71.8870772602 59.2669926152 80.352934115 47.8002988852 69.6470214273 44.044445022 69.495755496 58.785073162 68.6755160534 50.465234811 61.9080397046 53.1903520694 +32.1464623358 43.4817094425 64.8122375621 62.8109155873 73.0318330209 73.12057473 71.5385 60.641 76.2431578028 52.1236898503 67.2148371159 70.2600284356 75.3598819664 75.4216145799 73.7883730663 58.6208121164 60.3375864858 63.0349593272 77.392982494 63.3714590552 69.199057706 60.4189573202 69.0817099788 55.4740530987 71.8266529973 65.6862303604 +31.8384976954 71.8121653901 60.8536798727 65.4991470327 70.1028044999 75.6832068542 71.5385 64.4872 77.0844843669 51.9671367003 72.3020057291 70.2204373812 72.8336258709 72.1146938095 75.1396847343 54.2445521781 61.6587233282 64.1000234779 64.4007871926 67.4487184472 69.5510914402 58.4010982686 69.048716493 66.3660457477 72.6546231621 66.8782776113 +31.0052582572 40.682503007 67.7894961571 61.3637015467 57.1446946889 79.1941025922 69.4872 69.4872 75.2280531954 82.9856620201 66.7632573944 70.4738617113 70.8104148392 79.3504618327 64.5963189058 65.952512925 71.5450389858 72.657716331 63.8689598271 70.2137388333 69.5066843151 68.0844667893 67.3082233775 71.5902965807 76.8885781632 72.1654720777 +80.4708943189 49.5021483467 41.6095572691 83.8486865559 46.9624619942 78.6665589963 46.9231 79.8718 50.6583547714 83.1144793358 51.6803004439 89.9970275485 45.8019347617 81.6582677456 46.9095386323 85.6022611586 49.0409566406 77.3625832678 56.5944213157 86.9270062202 50.4441175504 75.0132995059 38.714899408 76.3969778172 43.7446196844 71.2679610751 +71.9641671122 41.8742826668 53.0030253202 84.6747986012 50.5649338128 79.0772985529 48.2051 84.1026 50.2033658114 82.9826568621 53.0965573571 89.540243712 45.835020171 82.7425919919 46.9051031994 81.5656424136 51.8868880222 80.1136076892 56.5313372853 87.4998110713 50.4672293846 90.281464505 35.5480948795 89.3049758407 39.0542484428 83.1190207043 +78.0794214417 93.1333167652 54.7141710591 84.0431280734 53.7196765696 79.306936167 50.0 85.2564 50.9295476993 82.8460411252 42.2422929389 89.7085785936 40.0788064496 80.2471852579 49.8069877356 82.085025592 57.968981435 85.3560457326 59.6521583697 87.8094615921 50.210881979 80.9417717084 36.7106729424 86.0749670003 39.9316746322 79.9391880033 +41.6775957748 30.2012640846 44.2916687157 82.9094412144 54.2002323226 79.3250023354 53.0769 85.2564 50.1786718465 83.1846223323 56.2338756577 89.635308157 51.9829247003 82.2043834675 48.8632418189 83.1583574877 54.855664646 83.7833976656 56.6365087005 85.637495556 50.205406955 90.6405359009 64.9126624396 87.6457960566 39.762826107 76.4059674459 +65.953595185 31.1474060835 49.1917219555 85.8762291224 47.875198343 78.6758534295 55.3846 86.0256 50.422698058 82.9099014671 54.6809865825 89.9645994589 57.337906053 84.9995188696 66.7481899024 84.0048341677 54.1331954613 83.7939960824 58.6722880025 90.077160307 50.4382906648 92.3547602661 66.0641357377 89.2086685815 41.323383287 80.2118685969 +62.9344593731 31.9163906992 53.1013817819 84.5476586881 51.4734585149 79.1598669963 56.6667 86.0256 50.4642248341 82.9353284131 57.9026866088 89.6184017294 62.3362327624 83.2465678988 62.3597662841 90.2436599196 51.3088252568 80.1363911553 58.2216127264 90.4110187715 50.3735188462 82.2192057218 64.9359294757 87.0721381174 41.7873232275 83.9880779935 +64.3737979844 28.8625834061 51.599848153 84.8198214898 52.7512222162 79.2936599344 56.1538 82.9487 50.4492703306 83.9699203811 51.8202625467 89.8341303987 58.5059100905 81.7455863409 47.3748964954 83.4715555182 61.1203943844 88.7040529069 57.9146644768 89.9538027677 50.253683689 81.7438287242 67.0265507063 76.2225153324 39.7556490651 79.6570966361 +72.5093283599 39.5401302526 54.3797219484 84.2403555494 48.5622230047 78.7823835275 53.8462 80.641 49.9283802752 82.9936654888 56.8283953362 89.5836647251 56.2479351377 83.513949493 47.4464667165 82.622124014 55.5638650304 85.8103751441 55.3155090581 80.2518606944 50.2504708592 71.9513852104 65.3140322257 77.9387440331 40.7498793985 77.204657286 +30.0522898741 96.6175423534 46.4807681047 83.518211666 53.0812416908 79.2972507654 51.2821 78.718 50.4880136436 83.0995191232 58.2722785867 69.9002729578 55.4065208295 80.2666719669 53.385280634 78.8711362558 55.044123361 83.5414106324 54.5757285877 77.5362884724 50.0870583609 73.9518736268 64.2952177764 84.1258572133 43.8568984983 77.6098436482 +28.0033242354 46.6721919544 53.174656268 84.2605679876 56.475383374 79.1678400979 50.0 78.718 49.9649053793 83.7083177013 51.8281274487 70.3526492053 53.2839688423 84.2635992213 49.4527077733 79.9306028004 50.4441853247 78.1958883115 54.4130936504 78.2290865878 50.2310718829 80.6473870602 38.7980121423 74.6106304299 42.3678224956 73.6382718294 +75.4012268619 88.6390766207 45.272002941 85.2370794195 49.570911405 79.0381238831 47.9487 77.5641 50.7521082559 82.9019500961 44.8499842618 70.3952029719 44.6767615455 80.2382855483 54.8026825363 75.3639849893 43.8277089981 71.0470522123 55.0745059041 79.8175464166 50.2077411201 73.1145035177 34.6757573144 79.6375987743 39.4629598364 78.459973898 +38.9800154218 87.322160691 36.03340215 53.371687755 37.6277032991 74.5575043418 29.7436 59.8718 27.4224292143 51.4388762325 24.2703823928 50.4855676918 30.0112642472 68.1467563743 37.0882451339 60.857227495 37.7914323802 66.8546410275 29.4329605156 60.8017765444 30.4995061735 53.6125276703 30.7192619427 58.812060082 44.5173339492 70.3693498252 +65.2199135479 84.6829549336 28.2711941653 72.840231257 32.2637000863 70.3166389012 29.7436 62.1795 27.6740833991 51.3041121452 34.7424714403 70.3399348879 29.7452539206 74.4045268347 32.3759619238 68.5108204552 29.292783111 59.79199693 29.4226860665 63.0684648229 30.5203612407 65.1004090686 34.1051509268 64.1291972267 41.6360929963 76.2492229205 +73.0539899616 29.3808085571 25.0548060847 71.5485979221 32.542380336 70.7024172063 31.2821 62.5641 27.5373953151 51.5936540762 37.7993149267 70.4695358763 37.3099483549 72.5861552003 37.4770138847 63.7886855744 32.1920969515 60.2006509147 29.0056141576 63.3907513334 30.4549558873 63.5017787814 33.4233718511 67.0957433613 45.4229321817 69.0955016709 +34.3983616372 59.5444469033 64.7588870014 82.3152236405 52.8250171923 85.5781338795 57.9487 99.4872 52.2633473764 94.2493278274 49.7869410649 89.6580469945 61.1359469093 82.5375693646 52.4066488967 96.4254150057 51.4339355233 99.2568672885 58.4618385916 90.2653263904 50.4164772292 97.2696919618 66.3138096741 98.9310270371 42.0028562742 97.8376147235 +43.4904501336 40.782542065 63.1445274767 85.2366963264 58.5282895407 78.9787042541 61.7949 99.1026 51.7372816586 92.9791175339 59.751497414 89.7319325216 61.7206191907 85.7056934492 50.2970432655 99.6441791736 67.4641398118 95.999963905 57.997804739 92.1599086113 69.5250693831 93.5980502089 65.3096314498 98.1651568396 76.1056290282 95.3049805754 +55.138737967 30.7257603575 50.4246786898 85.1747475891 56.265834263 79.2529394854 64.8718 97.5641 75.8709636929 88.3864417377 64.7747992444 90.132316736 63.6093798555 83.3600288362 61.5633243835 89.8641388605 65.1507052757 92.5026337368 57.5494740761 90.7489065581 69.5202566002 97.1938760167 65.7140760682 99.2837639519 79.2025626772 92.4072156264 +43.6843934333 32.8230098696 70.6449962629 82.4309187567 57.0758924193 79.1939225382 68.4615 94.1026 75.2443262052 83.9034973788 70.8444476004 89.9242090159 70.7237162322 87.1522075165 61.0865603919 89.2243556475 65.5834276962 93.5170609365 59.5299284566 88.3272741451 69.4989700668 92.1204899292 67.5934531877 90.9519098246 84.8482440854 95.4248045304 +35.9036097344 91.1118630801 63.1490490834 83.9468553485 54.2564023808 79.3138648504 70.7692 91.0256 75.1982952889 83.4623033365 70.6059455116 90.042061963 75.0063926418 82.4258351563 59.2364904011 91.4811005766 69.2889731377 99.5795911254 58.2493910631 92.1296814778 69.5286156637 91.6220882409 66.3880173202 89.2812304744 81.5644753207 83.7929072262 +45.3780188805 29.1692166544 62.8240245172 84.9661859507 62.6856712475 78.1747432719 72.0513 86.4103 75.7010415321 82.9194588559 78.6424883907 89.5512032307 75.0925497814 83.4373907092 66.1540050863 84.1119370252 81.2694405511 80.635944933 58.0245140126 91.6944211685 69.4997669273 84.47968329 67.8061881135 91.6881773501 85.4461864009 83.0782938448 +39.7774828713 43.75581895 70.236869515 82.1711510621 68.0856058616 76.1123175001 73.8462 83.3333 75.4793396586 82.8840513891 70.810692254 89.5926540118 70.3279291834 82.0293546968 67.2313347767 85.055640808 78.5757448361 76.8157030942 58.3821244904 90.5534760692 69.5038013697 81.8154056166 70.9626103129 86.5181213517 80.3124913784 76.5162389518 +38.6644611569 33.3172384774 70.0427352449 80.3850213547 65.1181219632 77.3991884543 75.1282 79.1026 75.1945668739 82.9321157846 66.7699466213 70.4233752108 70.8633308324 79.4081747457 73.7099068702 78.2706108976 74.1867734997 74.3442041467 62.5667590405 77.7439347591 69.4875893963 81.8359875567 71.9936371714 78.8209480695 80.7662322969 79.8960656796 +39.0440366877 84.6760108316 72.5706234458 80.9712184346 68.7456312971 75.7900554548 76.6667 75.2564 74.8202539551 82.9623887856 73.3363681612 70.3169271705 75.7799048195 79.8861994199 76.5804513576 72.8915336827 72.622209414 74.5269656511 72.1758243064 63.1289294176 69.5191157281 71.1159672213 72.4798343312 80.8006830564 82.6701124763 81.711479845 +91.6399614428 79.4066030605 75.1307160373 79.9840931403 63.0902251412 78.041409495 77.6923 71.4103 75.1643404933 83.0349971693 73.1051481627 70.300511881 78.0936590733 77.759558184 78.8765811639 71.5625276804 75.1415195918 73.8196684648 79.4727615693 63.4086861199 69.5043762947 70.4934438205 74.0784292286 72.9515308522 77.1636254828 73.5237189623 +47.4881326771 85.3899333808 83.2939057345 70.7784317876 77.2045893204 69.3121497658 79.7436 66.7949 75.2633555474 82.9452793018 74.6782541432 70.4453141903 76.7457612243 71.9733600919 77.4490021363 72.4473094857 75.168667839 73.9004448762 80.3577008812 63.2954375404 69.5125730109 61.497563947 72.0362880189 69.7160484161 74.0707000309 68.4795642895 +44.5902125769 22.0340116412 79.6642622775 73.9323097157 82.0339084758 62.6818697859 81.7949 60.2564 77.7564189331 51.1517703255 80.240250951 70.3543089613 78.4646034437 69.7493008209 86.8537073476 65.5433240194 85.3242028911 55.3885788252 78.7572361375 53.3326200111 89.4942005622 62.2343269676 72.9981862717 59.6656447092 74.1142308687 71.9263635185 +39.0896145478 70.4661940802 88.4321025254 64.6242470023 81.5779459183 63.8949255005 83.3333 55.2564 77.9544350549 50.4755789715 70.9468490729 50.4727017878 76.7456848486 70.0440572588 79.806531301 52.7231797883 84.090507598 56.301968006 82.5402395934 56.5410522935 89.5001240511 53.2163306165 74.8121312339 57.5898837631 68.942075458 62.635150577 +42.2293783752 19.9140684075 89.1155590082 64.0015066388 85.1801432805 53.7816766607 85.1282 51.4103 77.0833377719 52.1577992662 88.6626638086 50.4553260208 85.161682187 65.5124047088 76.9882734745 54.2027848022 86.2562238205 55.8309303793 86.4358971909 59.7927618087 89.490246254 48.890796551 73.0273050243 46.2741878725 73.2420023309 67.3532711288 +37.0003871448 60.264279248 89.0921976305 57.768193047 84.2868249936 57.2561680896 86.4103 47.5641 76.0635502513 52.1046520613 84.405952832 50.4948658866 91.8083525691 34.9655944844 79.4793575017 49.1410154455 92.5487933153 42.9019079116 79.4886844186 53.6516742641 89.49778562 46.6299066746 77.067111934 51.4869181896 71.8670131631 34.3291575887 +39.0520864793 70.6525028457 91.7260057727 52.6233532573 85.6224867126 45.0241658646 87.9487 46.0256 77.6820163247 51.1656378096 75.9871047068 50.4919669557 91.8874485924 34.7692538555 82.3041595613 46.9909036215 90.3426345089 37.7803861004 81.5304203188 56.0253645659 89.5048511306 48.4234081578 75.0171368727 46.6224426489 71.5691888532 34.699218607 +37.4884147432 60.8144048511 91.7355387598 48.9702108877 85.6044990156 50.7623379055 89.4872 42.5641 76.8780819814 51.8675622958 91.1120854191 50.489389393 92.2484016622 32.3771544295 90.700691247 41.4667030063 85.8191629042 37.5713861819 79.1867885665 53.2347918468 89.5010526463 43.3473436951 76.6653077326 38.440250966 71.4829096934 34.6805169926 +69.3595594592 65.5213545959 91.5078881747 53.3126520851 84.4247858276 38.761410164 93.3333 39.8718 76.9485027211 51.9075165439 98.2881232748 30.6039193719 91.782532741 34.3842946521 95.4434878077 36.1897016148 90.4418181137 40.1331134204 77.8990579454 51.8224583343 89.4888412549 40.6250912209 77.9158742329 45.9268433826 78.1285220512 23.139231835 +43.542775926 62.4603112824 88.239001904 31.4774348786 85.570985078 44.5600964397 95.3846 36.7949 77.8640547124 49.6625455341 95.0652748396 30.5004028652 96.0805193672 28.0536005018 94.9974880471 35.0953011066 92.0175590822 39.9179311324 75.1307142075 23.3724419733 89.5016235276 45.8151791904 73.7420460836 39.1209853231 78.789825254 25.5971933466 +39.8112302539 65.3348328092 88.5305192041 30.4760310098 84.6267242006 39.4870775084 98.2051 33.718 75.7714500936 17.1112512057 95.2492339568 30.4594542144 92.2478988278 28.0707514904 91.517847246 36.8902631263 95.2605278366 46.008830272 76.0580137544 16.3837496882 89.497529662 33.901372671 75.3298157891 32.8303519164 77.7815467962 23.4472144745 +70.0689259404 7.59346560899 55.3651603364 30.4458502777 73.3961549252 22.8162479426 56.6667 40.641 52.3315691301 51.8788603463 49.6561262881 50.4313198677 57.662276619 24.0606079328 62.2926421901 40.0295333141 41.2902998033 42.3115524113 57.6146743876 33.8224476502 50.4390920216 37.670771798 63.4104355141 38.377735614 48.5306293681 34.1524500077 +70.0405435824 77.1438066024 62.5602588694 30.4471356661 73.5074477338 22.8569887644 59.2308 38.3333 52.5928183721 51.3915915185 63.0186009279 30.4925819262 59.8469248931 26.1777124879 59.2034756073 41.2706776667 66.4313243161 36.5361097353 56.1713975295 32.1179887719 50.179850128 36.8075593352 68.856486669 43.084147214 65.5307814724 42.2810847015 +39.505789079 74.8516272173 58.0066691227 30.2537212987 63.3564881135 17.7240688424 60.7692 33.718 50.4770477213 17.0482894119 70.5382006004 30.4835582526 64.0670962137 27.7765005568 65.2563927704 32.8541041237 63.9863468858 32.7134316129 66.2878905956 26.1171097453 69.5050706711 33.7401525035 66.3377850519 33.3065100022 70.1180341426 36.4481847905 +62.5168908529 66.4847322418 55.0671179917 29.0115351997 72.9590468722 22.5114998325 63.0769 29.1026 75.2964750909 17.0156531888 70.0868697658 30.5100448173 60.9815013975 23.9342981582 62.4447424979 35.4149992261 61.2696820905 29.3857240482 67.8817196169 24.2360184109 69.5798670452 21.9352367455 64.2037185014 26.6441143003 70.4722626065 36.0880530553 +72.1399254065 0.0151193251552 61.6147759612 29.9943911942 64.0208416469 17.8835812521 64.1026 25.2564 75.573954129 17.0621921415 62.6856904424 30.5464245575 59.9670579092 26.8025832185 72.4624062907 27.6242638254 60.7493891909 28.0517363383 64.0280813016 27.6726855125 69.5064719492 22.7286628748 64.4986348346 22.863501327 76.7911300701 28.0720081685 +45.2515760666 70.0034213192 68.5431435378 35.6578321706 61.124463241 17.1163533592 64.359 24.1026 75.4005271583 17.0411068868 69.239417959 30.5060052447 62.7074541119 21.9955085787 69.9194262552 29.3110927191 64.6552305634 16.6122765086 77.4966517463 14.9485235577 69.4994788728 16.3374699821 68.8909906096 27.2962056672 72.4093545935 33.0392569664 +42.0633045627 2.33519661206 77.7061096487 20.3042601852 75.6742073297 24.7825009657 74.359 22.9487 75.8709908356 17.1348939075 80.4002336791 30.4850063746 69.1952352018 22.709792166 77.7628590782 23.4913735562 68.6382807138 18.5080128963 77.6346517616 14.461853927 69.500264751 27.1360239942 72.371523338 21.9616397473 76.271299815 25.5695441452 +36.3556951539 6.0058486497 68.4530459999 13.0355285908 69.5135831574 20.2793578927 71.2821 22.9487 75.6058847579 17.0677230553 74.6478782284 30.6738628584 65.5923626471 20.6941731019 72.7784101997 26.9005353695 65.7766651568 16.7876560655 77.8637264289 14.6106776491 69.5041565157 23.6689181006 69.7654249704 19.999850454 76.9532591384 27.7706107405 +30.3918276596 42.75961287 68.2572064402 12.3846391542 58.5721354669 16.642703808 67.9487 22.1795 75.8955770466 17.1699497051 65.9952727194 30.4750241915 64.6170985343 22.4431455531 65.198316667 33.0271717859 71.1341263421 19.3859575313 77.3381581659 15.890054658 69.5322795847 21.8509008151 68.6213124429 18.9156764428 75.1077300904 30.0169834157 +36.4490038543 50.5462690659 70.2554746739 13.2503849689 70.9819409865 21.1458881039 65.8974 20.2564 75.7465632023 17.1057165115 67.0885590036 30.4945239508 59.191159132 17.767901807 69.4598588916 30.1970315053 63.7231835039 14.5214211957 76.1804165329 15.9125737458 69.5001114942 11.7916260054 64.2977371716 20.4287496884 75.9692852004 29.0388653753 +40.467576002 60.0275120878 65.0443252778 11.0008414785 65.2230303973 18.3971951223 63.0769 19.1026 75.1423414811 16.7549238932 59.8525770283 10.5431877763 55.8396313724 21.6185696663 59.3150555205 12.3543136666 73.0014940701 24.0738125371 77.2526510945 15.1515170246 69.5048154243 15.6143582625 66.6992650902 18.5910853404 75.2838870131 24.8747531721 +81.7246168002 6.03754484635 60.0922466078 11.8721117093 65.3150439093 18.3337482986 61.2821 19.1026 50.6617795563 17.0781405153 53.7489900744 10.5225745323 59.0007586238 21.1710074885 51.487713355 20.8436106875 72.436657497 20.5783451136 77.4133752817 15.2219279762 69.5034745234 17.1690897689 67.5445276811 16.4479381344 77.0522406385 27.9056390894 +48.8231974964 76.6353305783 52.9920289667 9.90666848335 64.0102406778 17.9244781912 58.7179 18.3333 50.6998506392 17.0851843836 58.2390955164 10.4948990284 56.4593711284 25.1445642082 57.1124475509 15.3217851726 68.4127671225 19.9131265443 76.7318493982 16.2168461441 50.3774019862 16.5060150613 63.9469519836 18.6928454476 42.8267563374 25.5569594065 +35.6205617651 57.2860155789 50.1446289773 12.2115430946 57.1409359294 16.5986105437 55.1282 18.3333 50.9189408654 17.1476047571 54.810300495 10.7249292845 52.3581053558 19.4170541222 50.2244178157 19.1709698793 52.8583150204 20.2350610035 49.4711054118 25.0630193062 50.3394254813 18.7234506861 64.3881920705 15.7728122808 42.0806606852 24.9465264454 +50.5839631148 71.8066161014 46.5086141932 11.2071344881 56.3651532552 16.3265463742 52.3077 18.3333 50.7252585405 16.9074698083 52.9767123211 10.859490276 51.8116637782 17.0334945954 50.4338611907 18.5348708515 53.5299526734 24.2419155845 42.4765399431 18.3384735636 50.0936501016 11.6077134241 65.570045691 23.7657582226 39.8783046498 18.3249292157 +61.8564651063 71.7927431642 43.8070319612 11.3189448865 43.046183241 18.4025593948 49.7436 17.5641 51.2638712274 17.1623497098 52.2038939272 10.4956369212 54.222845621 19.503665544 56.3803054046 14.7884597101 50.7597410528 18.0893946004 43.5951158622 19.9942009773 49.9341271714 22.2561397401 38.4028368753 19.0468586722 36.4754991726 17.7189477884 +39.3237560262 59.3008196656 57.8178546865 10.9451424306 52.0817555582 16.4538745312 47.4359 16.0256 51.2509196468 17.2404558616 50.7451622429 10.5249055723 49.0187553996 23.4892740197 54.8635045999 14.009611915 42.53480844 10.1698684594 50.3399696652 26.4713966117 50.2888186122 13.436410466 37.8323600164 14.4694894463 37.1030736419 15.4456116477 +42.1856791429 66.0348978235 50.9404926595 9.69154713447 42.8790365154 18.4894110252 44.8718 13.718 50.7851572052 17.1801964842 42.7131936166 10.4875472428 35.6535812004 17.5145080595 45.4998576493 3.98542629287 41.5303978512 13.3962907452 40.7489802561 16.1821416622 50.2879944267 12.8722822086 36.9041611715 13.5838157511 34.1489682077 13.5238698677 +30.8469189898 37.3416401041 63.4973230835 11.9140691707 51.5669948027 16.4857136372 48.7179 14.8718 50.5013965822 17.1057707188 45.3172255098 10.4958733944 46.965785757 16.7892016247 49.9367453057 21.5900969196 46.7145179915 15.6132744651 38.3865255803 14.5802151464 50.2347207 20.4044729196 36.286143052 17.1057707266 37.5794219113 15.0158901333 +29.3462004281 42.1487418312 50.0164829541 11.9338520864 58.8391251216 16.6376352154 51.2821 14.8718 50.7336745372 16.9929634144 53.116915041 10.5068605973 49.5842392555 16.7579160147 53.8098729164 17.7664144512 48.4506176492 16.2122782224 38.4040152085 14.4519484496 50.2797016367 15.608737571 62.7866325947 13.9189931024 41.0643057894 12.2144689321 +82.1105579783 1.21055166293 58.6367650809 11.9747210652 60.7557891374 17.0332315064 54.1026 14.8718 50.7113785398 17.0883158538 57.049801823 10.5081369414 51.8562999655 21.969084007 59.8526782804 9.03994992133 59.6326101247 7.05803719901 38.7642788913 14.3655904744 50.2530067222 11.3020842468 66.8176792234 11.4124971575 42.195266147 20.9193721857 +38.3020058088 60.0177857932 54.7302890863 11.4128826732 54.7611561181 16.3661890954 56.1538 14.1026 50.8127448984 16.5727180506 54.8546309465 10.4639151924 55.9528886329 16.7897576048 48.1756567942 19.5212153543 61.6185682473 13.4108227184 41.4701423265 17.2780334445 50.4741411162 12.4619966676 66.755021412 18.0853051378 36.6055782383 15.748875885 +56.5841530218 70.512514809 65.8755478023 11.7324363627 53.5699050003 16.3397125506 52.0513 12.5641 51.014232955 17.2210955293 54.9248425638 10.498331942 51.6637862607 19.9251251977 50.7417190667 17.8384056087 48.8339891275 17.7259558674 47.1554048146 22.3779325323 50.1084399725 18.2590443442 65.4155283864 10.4635122068 39.4311612098 19.3238407541 +33.3393742865 0.5091067352 57.0609827131 9.92056085444 48.9146182416 16.7986370826 48.7179 11.0256 50.3535214052 17.0647430794 63.3095540306 10.4888142392 45.8596693166 17.1289036754 58.4597612975 14.1486628546 45.3426531889 13.9088123935 39.5825667453 17.6484528361 49.919058249 3.7168538855 36.9463314417 13.5143774996 37.5620763361 11.9284457044 +78.7742390407 35.4841012146 46.819907946 10.4946526804 52.6461373069 16.3848363836 47.1795 9.8718 50.4355295722 17.0651685025 49.7494984129 10.4858029097 39.7403795202 18.7410514146 50.1145975193 1.74146171252 42.4150122932 11.6937810961 41.7402438167 17.8293243101 50.0836645085 10.2358819024 37.8254347391 9.60103429534 34.2381100444 13.7131582305 +27.9226442446 25.9868781844 38.3593948725 13.431322624 49.8384929905 16.6763302777 46.1538 6.0256 50.6309819633 17.0765223454 41.1693104232 10.4973002433 52.8112216438 17.8549881176 45.4203820755 5.37240899542 58.9347650647 9.36920121911 39.3118718889 15.6407169743 50.3522608355 12.4686005273 36.722837943 9.33330210686 34.1409410386 4.57766135024 +71.6978651182 10.8681445111 47.3154157773 12.853451784 52.6840664009 16.4079627826 50.5128 9.4872 51.0668072036 17.2088597095 46.6883404976 10.5075321353 56.7891043639 17.3061155207 51.3532681954 0.30387242055 60.6836260407 10.6326415569 41.6798476925 17.7459190133 50.4975722411 7.40962523419 67.0733217806 6.04921458162 36.6592676972 17.6819644553 +74.1383313856 49.1739189791 55.0519165396 11.9499886181 65.8072190346 18.1407649473 53.8462 10.2564 50.7923547263 17.2042143387 52.4189219215 10.4959063596 54.1145317806 17.6735343896 53.091704424 19.2900304453 54.8117983683 6.56964870325 39.0874644519 15.1223039378 50.0163058346 16.2536847911 64.601818107 12.0019169643 40.6437192795 20.2408613716 +32.579020066 1.80811559665 50.5159602579 9.76559162187 60.6739209233 17.0518023644 57.4359 10.2564 50.5512780561 17.0846551833 50.8784883796 10.5030898746 55.571119732 17.369574581 58.9431665444 11.6128777918 61.5868424253 11.0275858356 41.4815028637 18.0474374446 50.4682212165 13.3567024555 65.4372821806 15.5453860785 36.7924815919 13.3921347304 +59.83218542 69.1525081443 49.6774146514 10.3831325083 63.394712866 17.7005351354 60.0 10.641 50.5597580562 17.0938837744 57.0533061394 10.5140692392 65.3064624975 18.1973548277 59.3109904219 10.5721363739 59.7160077628 11.5190968617 77.6060865492 15.1628725383 69.5506789842 9.03298756904 67.0040223792 15.3458266393 75.9483006427 22.7447959146 +35.0306285457 12.5366493416 67.2806595188 14.1286515323 64.8020094467 18.2168786335 64.1026 10.641 75.3259785471 15.7718919896 70.7864511597 10.5136950409 66.9907491546 17.2228706509 68.2221986595 1.13388036584 58.4163840114 5.64577748328 75.9826615205 16.3069223786 69.5186727475 15.1804864298 66.7241920702 5.24980548075 76.8406924717 20.9185472197 +74.3001198284 42.4770945921 66.1730182606 12.0379170217 65.0259398087 18.2878245002 66.9231 10.641 75.04472578 17.0042622562 56.7721495656 10.5032723508 72.1567189391 16.8142747914 62.5117719862 7.49123248279 59.243809528 10.5401420943 76.9457572424 15.858478334 69.5034574098 16.8783132661 68.3076187632 13.2809165227 81.9504776395 19.8745582085 +63.2501970628 65.9524861966 61.0885441374 10.0845344144 65.7555515404 18.5162158303 71.2821 10.641 75.2870877232 16.1749349097 75.4772220231 10.5185680355 70.9805804021 17.15575228 69.3800563617 1.48813233299 56.5943646385 8.91734404523 77.5437200743 15.253949149 69.4994868291 10.2078266798 68.7680482759 13.5214565521 81.6437056853 16.5917599845 +34.1730737648 25.6936743092 66.0530857731 13.3802260073 69.9677341245 20.5389887758 74.359 10.641 75.2399699777 17.0318474866 76.7659121498 10.496999064 72.2220822073 19.3692816955 82.8002478704 18.5293762368 68.4925132247 17.9873634289 77.5847398447 15.8300393854 69.5275296829 9.99229666551 74.1672717461 5.34988087254 84.6102197758 14.2477619017 +40.9138319319 38.5590195509 72.6699892655 15.2342259409 68.8927840241 20.0324488429 78.2051 10.641 75.1524592023 17.0049424015 77.621049129 10.4958742318 79.7001583481 20.740218365 85.1603321777 15.0183695439 65.0059431717 15.9126686464 76.8223042562 15.5951653195 69.5190952617 11.4226857018 64.9003579699 16.2452583689 84.1028794336 8.98065579006 +62.8332930874 62.5108942269 61.503472503 10.8284144754 61.8265465579 17.2696554452 67.9487 8.718 75.9618400861 16.6948422289 65.4144870523 10.6818401251 64.2491606461 14.9139624607 67.095847697 0.601490941965 63.9210367078 15.6209610332 77.3485716647 15.7745292395 69.5618994268 15.9998037858 68.7634335463 8.70057294385 80.18002545 19.0888441297 +42.4767923803 56.7312899691 68.995028632 13.9943105293 60.7879700808 17.0462048583 68.4615 5.2564 75.4480625064 17.0451494125 69.9647805002 10.5072329326 66.9432914334 20.1891325034 62.307750573 5.74997471945 53.5129705438 5.87510952836 77.5731526867 14.7806458332 69.5010445197 5.95583353945 66.8169139163 12.2732943361 80.5732729943 8.36563890168 +52.0334562787 64.5666620298 78.2499161658 17.8832409068 61.9156096372 17.2821912228 68.2051 2.9487 75.7593838168 16.9429296545 64.9697382623 10.4821595117 66.9416766634 18.2435093914 59.9704150458 7.9133104735 57.8328062743 9.22238462333 77.9726106776 14.9556987461 69.524635976 2.73476016988 67.3093466467 0.217006270014 79.0043268649 10.6281977654 +48.9070429644 74.2877488252 36.4819805699 15.1627600944 45.0289975331 17.7244473049 37.6923 25.7692 50.378262298 16.9462798113 38.0502407785 30.4079460343 39.6482666468 22.7313663718 39.9116351739 32.1440226145 31.0329174395 28.9281418896 41.5289197586 24.9164251896 30.4995937106 26.0855615691 34.7318293093 19.6017950724 40.0481864651 24.2614879334 +59.8518383854 72.9583909677 50.9677483784 29.6797742875 39.9213628084 19.7018495281 39.4872 25.3846 50.5336350094 17.0195813691 42.2241202189 30.4980371582 37.9497836858 26.5217492344 39.5965009783 32.1361317349 31.3332440932 32.5385694785 43.7225508034 19.0773278036 30.487391709 19.7794703733 33.6744421497 26.09049021 34.7945935378 13.9696834611 +48.9604602016 72.6295257275 91.1910536059 46.6743428399 84.7942778751 55.5686503735 91.2821 41.5385 77.5009073165 50.1669859475 87.6984642949 50.4899365296 86.5043905611 34.8940252874 95.5934163965 33.2341293953 86.4015498466 38.7469331743 79.3260781751 52.9003912897 89.5001804719 31.9789174357 75.6272550236 37.128751949 79.221764441 22.0945914837 +46.8448551673 36.7917136918 55.8637684939 85.3364867573 55.6629587449 83.3564796243 50.0 95.7692 50.6911241884 87.5139604241 52.3603310701 89.7043170068 50.8154853559 78.950838805 52.772288113 99.6134716833 47.4421116354 98.1843018589 56.6639740815 87.9401250067 50.4102716314 98.6283694405 40.6101254225 89.1362398699 36.0308797708 93.1217332374 +39.9630215796 42.9449148487 49.2805947987 84.0488228306 50.4922475132 78.997532065 47.9487 95.0 49.9903949511 83.997356924 51.4067867898 89.7121385499 51.1834666638 85.1288546497 53.1786144535 98.6038611855 46.2647413368 94.1161921613 57.8217892319 90.6931665489 50.3259243927 94.9946308528 39.1143664549 96.4817513202 34.4995583102 86.6099850511 +66.704943997 32.0150954299 43.3685015447 84.3321772039 51.4671011416 79.2018445773 44.1026 92.6923 50.1271820272 82.990749996 43.588468468 89.7455365536 40.8638552293 82.9261452983 37.6270534612 93.060672952 40.1638157437 87.4486723469 58.2431718991 92.1043278652 50.1040315041 95.0885380179 34.5838289299 89.5889019877 31.1068665551 89.4616352367 diff --git a/example/neural_spline_flow/data/data_proc.jl b/example/neural_spline_flow/data/data_proc.jl new file mode 100644 index 00000000..8d572ca1 --- /dev/null +++ b/example/neural_spline_flow/data/data_proc.jl @@ -0,0 +1,7 @@ +using CSV, DataFrames + +# Read the .tsv file +dat = CSV.read("example/neural_spline_flow/data/DatasaurusDozen-Wide.tsv", DataFrame) +dat_dino = permutedims(hcat(dat.dino, dat.dino_1)[2:end, :]) +dat_dino = map(x -> parse(Float32, x), dat_dino) +dat = vcat(dat_dino, dat_dino) diff --git a/src/objectives/loglikelihood.jl b/src/objectives/loglikelihood.jl index 1b1b29e5..fc2f7369 100644 --- a/src/objectives/loglikelihood.jl +++ b/src/objectives/loglikelihood.jl @@ -32,7 +32,7 @@ function loglikelihood( N = length(fulldata) @assert batchsize <= N idx = sample(rng, 1:N, batchsize; replace=false) - xs = @view(fulldata[idx]) + xs = fulldata[idx] return loglikelihood(flow, xs) end @@ -45,6 +45,6 @@ function loglikelihood( N = size(fulldata, 2) @assert batchsize <= N idx = sample(rng, 1:N, batchsize; replace=false) - xs = @view(fulldata[:, idx]) + xs = fulldata[:, idx] return loglikelihood(flow, xs) end \ No newline at end of file From 2dd855a92f93be2f7c9c76817d57d072625065a3 Mon Sep 17 00:00:00 2001 From: zuhengxu Date: Wed, 8 Jan 2025 16:04:45 -0800 Subject: [PATCH 40/40] rm readme --- README.md | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index 9ab359dd..00000000 --- a/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# NormalizingFlows.jl - -[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://turinglang.github.io/NormalizingFlows.jl/dev/) -[![Build Status](https://github.com/TuringLang/NormalizingFlows.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/TuringLang/NormalizingFlows.jl/actions/workflows/CI.yml?query=branch%3Amain) - - -A normalizing flow library for Julia. - -The purpose of this package is to provide a simple and flexible interface for -variational inference (VI) and normalizing flows (NF) for Bayesian computation or generative modeling. -The key focus is to ensure modularity and extensibility, so that users can easily -construct (e.g., define customized flow layers) and combine various components -(e.g., choose different VI objectives or gradient estimates) -for variational approximation of general target distributions, -without being tied to specific probabilistic programming frameworks or applications. - -See the [documentation](https://turinglang.org/NormalizingFlows.jl/dev/) for more. - -## Installation -To install the package, run the following command in the Julia REPL: -```julia -# install the package -] # enter Pkg mode -(@v1.9) pkg> add git@github.com:TuringLang/NormalizingFlows.jl.git -``` -Then simply run the following command to use the package: -```julia -using NormalizingFlows -``` - - -## Quick recap of normalizing flows - - -## Current status and TODOs - -