Merge pull request JuliaLogging#129 from JamieMair/add-hparams-api

Add hparams API

Author: nomadbl

.gitignore

@@ -6,4 +6,4 @@ test/test_logs
 docs/Manifest.toml
 
 gen/proto
-gen/protojl
+gen/protojl

Project.toml

@@ -20,16 +20,16 @@ StatsBase = "0.27, 0.28, 0.29, 0.30, 0.31, 0.32, 0.33, 0.34"
 julia = "1.6"
 
 [extras]
+Cairo = "159f3aea-2a34-519c-b102-8c37f9878175"
+Fontconfig = "186bb1d3-e1f7-5a2c-a377-96d770f13627"
+Gadfly = "c91e804a-d5a3-530f-b6f0-dfbca275c004"
 ImageMagick = "6218d12a-5da1-5696-b52f-db25d2ecc6d1"
 LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458"
 Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee"
-Gadfly = "c91e804a-d5a3-530f-b6f0-dfbca275c004"
-Cairo="159f3aea-2a34-519c-b102-8c37f9878175"
-Fontconfig="186bb1d3-e1f7-5a2c-a377-96d770f13627"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestImages = "5e47fb64-e119-507b-a336-dd2b206d9990"
 Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"

docs/make.jl

@@ -10,11 +10,13 @@ makedocs(
                 "Backends"     =>  "custom_behaviour.md",
                 "Reading back data" => "deserialization.md",
                 "Extending"    =>  "extending_behaviour.md",
-                "Explicit Interface" => "explicit_interface.md"
+                "Explicit Interface" => "explicit_interface.md",
+                "Hyperparameter logging" => "hyperparameters.md"
             ],
             "Examples"      => Any[
                 "Flux.jl"  => "examples/flux.md"
                 "Optim.jl"  => "examples/optim.md"
+                "Hyperparameter tuning"  => "examples/hyperparameter_tuning.md"
             ]
     ],
     format    = Documenter.HTML(

docs/src/examples/hyperparameter_tuning.md

@@ -0,0 +1,53 @@
+# Hyperparameter tuning
+
+We will start this example by setting up a simple random walk experiment, and seeing the effect of the hyperparameter `bias` on the results.
+
+First, import the packages we will need with:
+```julia
+using TensorBoardLogger, Logging
+using Random
+```
+Next, we will create a function which runs the experiment and logs the results, include the hyperparameters stored in the `config` dictionary.
+```julia
+function run_experiment(id, config)
+    logger = TBLogger("random_walk/run$id", tb_append)
+
+    # Specify all the metrics we want to track in a list
+    metric_names = ["scalar/position"]
+    write_hparams!(logger, config, metric_names)
+
+    epochs = config["epochs"]
+    sigma = config["sigma"]
+    bias = config["bias"]
+    with_logger(logger) do
+        x = 0.0
+        for i in 1:epochs
+            x += sigma * randn() + bias
+            @info "scalar" position = x
+        end
+    end
+    nothing
+end
+```
+Now we can write a script which runs an experiment over a set of parameter values.
+```julia
+id = 0
+for bias in LinRange(-0.1, 0.1, 11)
+    for epochs in [50, 100]
+        config = Dict(
+            "bias"=>bias,
+            "epochs"=>epochs,
+            "sigma"=>0.1
+        )
+        run_experiment(id, config)
+        id += 1
+    end
+end
+```
+
+Below is an example of the dashboard you get when you open Tensorboard with the command:
+```sh
+tensorboard --logdir=random_walk
+```
+
+![tuning plot](tuning.png)

docs/src/examples/tuning.png

@@ -0,0 +1,10 @@
+# Hyperparameter logging
+
+In additition to logging the experiments, you may wish to also visualise the effect of hyperparameters on some plotted metrics. This can be done by logging the hyperparameters via the `write_hparams!` function, which takes a dictionary mapping hyperparameter names to their values (currently limited to `Real`, `Bool` or `String` types), along with the names of any metrics that you want to view the effects of. 
+
+You can see how the HParams dashboard in Tensorboard can be used to tune hyperparameters on the [tensorboard website](https://www.tensorflow.org/tensorboard/hyperparameter_tuning_with_hparams).
+
+## API
+```@docs
+write_hparams!
+```

docs/src/hyperparameters.md

@@ -117,6 +117,9 @@ We also support logging custom types from a the following third-party libraries:
 
 ## Explicit logging
 
-In alternative, you can also log data to TensorBoard through its functional interface,
-by calling the relevant method with a tag string and the data. For information
-on this interface refer to [Explicit interface](@ref)...
+As an alternative, you can also log data to TensorBoard through its functional interface, by calling the relevant method with a tag string and the data. For information on this interface refer to [Explicit interface](@ref).
+
+## Hyperparameter tuning
+
+Many experiments rely on hyperparameters, which can be difficult to tune. Tensorboard allows you to visualise the effect of your hyperparameters on your metrics, giving you an intuition for the correct hyperparameters for your task. For information on this API, see the [Hyperparameter logging](@ref) manual page.
+

docs/src/index.md

@@ -0,0 +1,38 @@
+using TensorBoardLogger #import the TensorBoardLogger package
+using Logging #import Logging package
+using Random # Exports randn
+
+# Run 10 experiments to see a plot
+for j in 1:10
+    logger = TBLogger("random_walks/run$j", tb_append)
+
+    sigma = 0.1
+    epochs = 200
+    bias = (rand()*2 - 1) / 10 # create a random bias
+    use_seed = false
+    # Add in the a dummy loss metric
+    with_logger(logger) do
+        x = 0.0
+        for i in 1:epochs
+            x += sigma * randn() + bias
+            @info "scalar" loss = x
+        end
+    end
+
+    # Hyperparameter is a dictionary of parameter names to their values. This 
+    # supports numerical types, bools and strings. Non-bool numerical types 
+    # are converted to Float64 to be displayed.
+    hparams_config = Dict{String, Any}(
+        "sigma"=>sigma,
+        "epochs"=>epochs,
+        "bias"=>bias,
+        "use_seed"=>use_seed,
+        "method"=>"MC"
+    )
+    # Specify a list of tags that you want to show up in the hyperparameter 
+    # comparison
+    metrics = ["scalar/loss"]
+    
+    # Write the hyperparameters and metrics config to the logger.
+    write_hparams!(logger, hparams_config, metrics)
+end

examples/HParams.jl

@@ -4,5 +4,5 @@ FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429"
 
-[comapt]
+[compat]
 ProtoBuf = "0.9.1"

gen/Project.toml

@@ -21,7 +21,7 @@ using  Base.CoreLogging: CoreLogging, AbstractLogger, LogLevel, Info,
 
 export TBLogger, reset!, set_step!, increment_step!, set_step_increment!, with_TBLogger_hold_step
 export log_histogram, log_value, log_vector, log_text, log_image, log_images,
-       log_audio, log_audios, log_graph, log_embeddings, log_custom_scalar
+       log_audio, log_audios, log_graph, log_embeddings, log_custom_scalar, write_hparams!
 export map_summaries, TBReader
 
 export ImageFormat, L, CL, LC, LN, NL, NCL, NLC, CLN, LCN, HW, WH, HWC, WHC,
@@ -62,6 +62,7 @@ include("ImageFormat.jl")
 const TB_PLUGIN_JLARRAY_NAME = "_jl_tbl_array_sz"
 
 include("TBLogger.jl")
+include("hparams.jl")
 include("utils.jl")  # CRC Utils
 include("event.jl")
 include("Loggers/base.jl")