Safe Haskell	Safe-Inferred
Language	Haskell2010

Development.Benchmark.Rules

Description

This module provides a bunch of Shake rules to build multiple revisions of a project and analyse their performance.

It assumes a project bench suite composed of examples that runs a fixed set of experiments on every example

Your code must implement all of the GetFoo oracles and the IsExample class, instantiate the Shake rules, and probably want a set of targets.

The results of the benchmarks and the analysis are recorded in the file system, using the following structure:

build-folder ├── binaries │ └── git-reference │ ├── ghc.path - path to ghc used to build the executable │ ├── executable - binary for this version │ └── commitid - Git commit id for this reference ├─ example │ ├── results.csv - aggregated results for all the versions and configurations │ ├── experiment.svg - graph of bytes over elapsed time, for all the versions and configurations | └── git-reference │ └── configuration │ ├── experiment.gcStats.log - RTS -s output │ ├── experiment.csv - stats for the experiment │ ├── experiment.svg - Graph of bytes over elapsed time │ ├── experiment.diff.svg - idem, including the previous version │ ├── experiment.heap.svg - Heap profile │ ├── experiment.log - bench stdout │ └── results.csv - results of all the experiments for the example ├── results.csv - aggregated results of all the examples, experiments, versions and configurations └── experiment.svg - graph of bytes over elapsed time, for all the examples, experiments, versions and configurations

For diff graphs, the "previous version" is the preceding entry in the list of versions in the config file. A possible improvement is to obtain this info via `git rev-list`.

Synopsis

buildRules :: FilePattern -> MkBuildRules BuildSystem -> Rules ()
data MkBuildRules buildSystem = MkBuildRules {
- findGhc :: buildSystem -> FilePath -> IO FilePath
- executableName :: String
- projectDepends :: Action ()
- buildProject :: buildSystem -> ProjectRoot -> OutputFolder -> Action ()
}
type OutputFolder = FilePath
type ProjectRoot = FilePath
benchRules :: RuleResultForExample example => FilePattern -> MkBenchRules BuildSystem example -> Rules ()
data MkBenchRules buildSystem example = forall setup.MkBenchRules {
- setupProject :: Action setup
- benchProject :: setup -> buildSystem -> [CmdOption] -> BenchProject example -> Action ()
- warmupProject :: buildSystem -> FilePath -> [CmdOption] -> example -> Action ()
- executableName :: String
- parallelism :: Natural
}
data BenchProject example = BenchProject {
- outcsv :: FilePath
- exePath :: FilePath
- exeExtraArgs :: [String]
- example :: example
- experiment :: Escaped String
- configuration :: ByteString
}
data ProfilingMode
- = NoProfiling
- | CheapHeapProfiling Seconds
csvRules :: forall example. RuleResultForExample example => FilePattern -> Rules ()
svgRules :: FilePattern -> Rules ()
heapProfileRules :: FilePattern -> Rules ()
phonyRules :: (Traversable t, IsExample e) => String -> String -> ProfilingMode -> FilePath -> t e -> Rules ()
allTargetsForExample :: IsExample e => ProfilingMode -> FilePath -> e -> Action [FilePath]
newtype GetExample = GetExample String
newtype GetExamples = GetExamples ()
class (Binary e, Eq e, Hashable e, NFData e, Show e, Typeable e) => IsExample e where
- getExampleName :: e -> String
type RuleResultForExample e = (RuleResult GetExample ~ Maybe e, RuleResult GetExamples ~ [e], IsExample e)
newtype GetExperiments = GetExperiments ()
newtype GetVersions = GetVersions ()
newtype GetCommitId = GetCommitId String
newtype GetBuildSystem = GetBuildSystem ()
newtype GetConfigurations = GetConfigurations ()
data Configuration = Configuration {
- confName :: String
- confValue :: ByteString
}
data BuildSystem
- = Cabal
- | Stack
findGhcForBuildSystem :: BuildSystem -> FilePath -> IO FilePath
newtype Escaped a = Escaped {
- escaped :: a
}
newtype Unescaped a = Unescaped {
- unescaped :: a
}
escapeExperiment :: Unescaped String -> Escaped String
unescapeExperiment :: Escaped String -> Unescaped String
data GitCommit

Documentation

buildRules :: FilePattern -> MkBuildRules BuildSystem -> Rules () Source #

Rules that drive a build system to build various revisions of a project

data MkBuildRules buildSystem Source #

Constructors

MkBuildRules

Fields

findGhc :: buildSystem -> FilePath -> IO FilePath
Return the path to the GHC executable to use for the project found in the cwd
executableName :: String
Name of the binary produced by $sel:buildProject:MkBuildRules
projectDepends :: Action ()
An action that captures the source dependencies, used for the HEAD build
buildProject :: buildSystem -> ProjectRoot -> OutputFolder -> Action ()
Build the project found in the given path and save the build artifacts in the output folder

type OutputFolder = FilePath Source #

type ProjectRoot = FilePath Source #

benchRules :: RuleResultForExample example => FilePattern -> MkBenchRules BuildSystem example -> Rules () Source #

data MkBenchRules buildSystem example Source #

Constructors

forall setup. MkBenchRules

Fields

setupProject :: Action setup
Workaround for Shake not allowing to call askOracle from 'benchProject
benchProject :: setup -> buildSystem -> [CmdOption] -> BenchProject example -> Action ()
An action that invokes the executable to run the benchmark
warmupProject :: buildSystem -> FilePath -> [CmdOption] -> example -> Action ()
An action that performs any necessary warmup. Will only be invoked once
executableName :: String
Name of the executable to benchmark. Should match the one used to MkBuildRules
parallelism :: Natural
Number of concurrent benchmarks to run

data BenchProject example Source #

Constructors

BenchProject
Fields outcsv :: FilePath where to save the CSV output exePath :: FilePath where to find the executable for benchmarking exeExtraArgs :: [String] extra args for the executable example :: example example to benchmark experiment :: Escaped String experiment to run configuration :: ByteString configuration to use

data ProfilingMode Source #

Constructors

NoProfiling
CheapHeapProfiling Seconds

Instances

Instances details

Eq ProfilingMode Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: ProfilingMode -> ProfilingMode -> Bool # (/=) :: ProfilingMode -> ProfilingMode -> Bool #

csvRules :: forall example. RuleResultForExample example => FilePattern -> Rules () Source #

Rules to aggregate the CSV output of individual experiments

svgRules :: FilePattern -> Rules () Source #

Rules to produce charts for the GC stats

heapProfileRules :: FilePattern -> Rules () Source #

phonyRules Source #

Arguments

:: (Traversable t, IsExample e)
=> String	prefix
-> String	Executable name
-> ProfilingMode
-> FilePath
-> t e
-> Rules ()

Generate a set of phony rules: * prefixall * prefix example for each example

allTargetsForExample :: IsExample e => ProfilingMode -> FilePath -> e -> Action [FilePath] Source #

newtype GetExample Source #

Constructors

GetExample String

Instances

Instances details

Show GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetExample -> ShowS # show :: GetExample -> String # showList :: [GetExample] -> ShowS #
Binary GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetExample -> Put # get :: Get GetExample # putList :: [GetExample] -> Put #
NFData GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetExample -> () #
Eq GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetExample -> GetExample -> Bool # (/=) :: GetExample -> GetExample -> Bool #
Hashable GetExample Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetExample -> Int # hash :: GetExample -> Int #

newtype GetExamples Source #

Constructors

GetExamples ()

Instances

Instances details

Show GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetExamples -> ShowS # show :: GetExamples -> String # showList :: [GetExamples] -> ShowS #
Binary GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetExamples -> Put # get :: Get GetExamples # putList :: [GetExamples] -> Put #
NFData GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetExamples -> () #
Eq GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetExamples -> GetExamples -> Bool # (/=) :: GetExamples -> GetExamples -> Bool #
Hashable GetExamples Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetExamples -> Int # hash :: GetExamples -> Int #

class (Binary e, Eq e, Hashable e, NFData e, Show e, Typeable e) => IsExample e where Source #

Knowledge needed to run an example

Methods

getExampleName :: e -> String Source #

type RuleResultForExample e = (RuleResult GetExample ~ Maybe e, RuleResult GetExamples ~ [e], IsExample e) Source #

newtype GetExperiments Source #

Constructors

GetExperiments ()

Instances

Instances details

Show GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetExperiments -> ShowS # show :: GetExperiments -> String # showList :: [GetExperiments] -> ShowS #
Binary GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetExperiments -> Put # get :: Get GetExperiments # putList :: [GetExperiments] -> Put #
NFData GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetExperiments -> () #
Eq GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetExperiments -> GetExperiments -> Bool # (/=) :: GetExperiments -> GetExperiments -> Bool #
Hashable GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetExperiments -> Int # hash :: GetExperiments -> Int #
type RuleResult GetExperiments Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetExperiments = [Unescaped String]

newtype GetVersions Source #

Constructors

GetVersions ()

Instances

Instances details

Show GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetVersions -> ShowS # show :: GetVersions -> String # showList :: [GetVersions] -> ShowS #
Binary GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetVersions -> Put # get :: Get GetVersions # putList :: [GetVersions] -> Put #
NFData GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetVersions -> () #
Eq GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetVersions -> GetVersions -> Bool # (/=) :: GetVersions -> GetVersions -> Bool #
Hashable GetVersions Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetVersions -> Int # hash :: GetVersions -> Int #
type RuleResult GetVersions Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetVersions = [GitCommit]

newtype GetCommitId Source #

Constructors

GetCommitId String

Instances

Instances details

Show GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetCommitId -> ShowS # show :: GetCommitId -> String # showList :: [GetCommitId] -> ShowS #
Binary GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetCommitId -> Put # get :: Get GetCommitId # putList :: [GetCommitId] -> Put #
NFData GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetCommitId -> () #
Eq GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetCommitId -> GetCommitId -> Bool # (/=) :: GetCommitId -> GetCommitId -> Bool #
Hashable GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetCommitId -> Int # hash :: GetCommitId -> Int #
type RuleResult GetCommitId Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetCommitId = String

newtype GetBuildSystem Source #

Constructors

GetBuildSystem ()

Instances

Instances details

Show GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetBuildSystem -> ShowS # show :: GetBuildSystem -> String # showList :: [GetBuildSystem] -> ShowS #
Binary GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetBuildSystem -> Put # get :: Get GetBuildSystem # putList :: [GetBuildSystem] -> Put #
NFData GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetBuildSystem -> () #
Eq GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetBuildSystem -> GetBuildSystem -> Bool # (/=) :: GetBuildSystem -> GetBuildSystem -> Bool #
Hashable GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetBuildSystem -> Int # hash :: GetBuildSystem -> Int #
type RuleResult GetBuildSystem Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetBuildSystem = BuildSystem

newtype GetConfigurations Source #

Constructors

GetConfigurations ()

Instances

Instances details

Show GetConfigurations Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GetConfigurations -> ShowS # show :: GetConfigurations -> String # showList :: [GetConfigurations] -> ShowS #
Binary GetConfigurations Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GetConfigurations -> Put # get :: Get GetConfigurations # putList :: [GetConfigurations] -> Put #
NFData GetConfigurations Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GetConfigurations -> () #
Eq GetConfigurations Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GetConfigurations -> GetConfigurations -> Bool # (/=) :: GetConfigurations -> GetConfigurations -> Bool #
Hashable GetConfigurations Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GetConfigurations -> Int # hash :: GetConfigurations -> Int #
type RuleResult GetConfigurations Source #
Instance details Defined in Development.Benchmark.Rules type RuleResult GetConfigurations = [Configuration]

data Configuration Source #

Constructors

Configuration
Fields confName :: String confValue :: ByteString

Instances

Instances details

Generic Configuration Source #
Instance details Defined in Development.Benchmark.Rules Associated Types type Rep Configuration :: Type -> Type # Methods from :: Configuration -> Rep Configuration x # to :: Rep Configuration x -> Configuration #
Show Configuration Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> Configuration -> ShowS # show :: Configuration -> String # showList :: [Configuration] -> ShowS #
Binary Configuration Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: Configuration -> Put # get :: Get Configuration # putList :: [Configuration] -> Put #
NFData Configuration Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: Configuration -> () #
Eq Configuration Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: Configuration -> Configuration -> Bool # (/=) :: Configuration -> Configuration -> Bool #
Hashable Configuration Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> Configuration -> Int # hash :: Configuration -> Int #
type Rep Configuration Source #
Instance details Defined in Development.Benchmark.Rules type Rep Configuration = D1 ('MetaData "Configuration" "Development.Benchmark.Rules" "shake-bench-0.2.0.0-GBfwpyOoQh65kxC6vx003f" 'False) (C1 ('MetaCons "Configuration" 'PrefixI 'True) (S1 ('MetaSel ('Just "confName") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 String) :*: S1 ('MetaSel ('Just "confValue") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 ByteString)))

data BuildSystem Source #

Default build system that handles Cabal and Stack

Constructors

Cabal
Stack

Instances

Instances details

FromJSON BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods parseJSON :: Value -> Parser BuildSystem # parseJSONList :: Value -> Parser [BuildSystem] #
ToJSON BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods toJSON :: BuildSystem -> Value # toEncoding :: BuildSystem -> Encoding # toJSONList :: [BuildSystem] -> Value # toEncodingList :: [BuildSystem] -> Encoding #
Generic BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Associated Types type Rep BuildSystem :: Type -> Type # Methods from :: BuildSystem -> Rep BuildSystem x # to :: Rep BuildSystem x -> BuildSystem #
Read BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods readsPrec :: Int -> ReadS BuildSystem # readList :: ReadS [BuildSystem] # readPrec :: ReadPrec BuildSystem # readListPrec :: ReadPrec [BuildSystem] #
Show BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> BuildSystem -> ShowS # show :: BuildSystem -> String # showList :: [BuildSystem] -> ShowS #
Binary BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: BuildSystem -> Put # get :: Get BuildSystem # putList :: [BuildSystem] -> Put #
NFData BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: BuildSystem -> () #
Eq BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: BuildSystem -> BuildSystem -> Bool # (/=) :: BuildSystem -> BuildSystem -> Bool #
Hashable BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> BuildSystem -> Int # hash :: BuildSystem -> Int #
type Rep BuildSystem Source #
Instance details Defined in Development.Benchmark.Rules type Rep BuildSystem = D1 ('MetaData "BuildSystem" "Development.Benchmark.Rules" "shake-bench-0.2.0.0-GBfwpyOoQh65kxC6vx003f" 'False) (C1 ('MetaCons "Cabal" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Stack" 'PrefixI 'False) (U1 :: Type -> Type))

findGhcForBuildSystem :: BuildSystem -> FilePath -> IO FilePath Source #

newtype Escaped a Source #

Constructors

Escaped
Fields escaped :: a

newtype Unescaped a Source #

Constructors

Unescaped
Fields unescaped :: a

Instances

Instances details

FromJSON a => FromJSON (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods parseJSON :: Value -> Parser (Unescaped a) # parseJSONList :: Value -> Parser [Unescaped a] #
ToJSON a => ToJSON (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods toJSON :: Unescaped a -> Value # toEncoding :: Unescaped a -> Encoding # toJSONList :: [Unescaped a] -> Value # toEncodingList :: [Unescaped a] -> Encoding #
Show a => Show (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> Unescaped a -> ShowS # show :: Unescaped a -> String # showList :: [Unescaped a] -> ShowS #
Binary a => Binary (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: Unescaped a -> Put # get :: Get (Unescaped a) # putList :: [Unescaped a] -> Put #
NFData a => NFData (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: Unescaped a -> () #
Eq a => Eq (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: Unescaped a -> Unescaped a -> Bool # (/=) :: Unescaped a -> Unescaped a -> Bool #
Hashable a => Hashable (Unescaped a) Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> Unescaped a -> Int # hash :: Unescaped a -> Int #

escapeExperiment :: Unescaped String -> Escaped String Source #

unescapeExperiment :: Escaped String -> Unescaped String Source #

data GitCommit Source #

Instances

Instances details

FromJSON GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods parseJSON :: Value -> Parser GitCommit # parseJSONList :: Value -> Parser [GitCommit] #
ToJSON GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods toJSON :: GitCommit -> Value # toEncoding :: GitCommit -> Encoding # toJSONList :: [GitCommit] -> Value # toEncodingList :: [GitCommit] -> Encoding #
Generic GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Associated Types type Rep GitCommit :: Type -> Type # Methods from :: GitCommit -> Rep GitCommit x # to :: Rep GitCommit x -> GitCommit #
Show GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods showsPrec :: Int -> GitCommit -> ShowS # show :: GitCommit -> String # showList :: [GitCommit] -> ShowS #
Binary GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods put :: GitCommit -> Put # get :: Get GitCommit # putList :: [GitCommit] -> Put #
NFData GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods rnf :: GitCommit -> () #
Eq GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods (==) :: GitCommit -> GitCommit -> Bool # (/=) :: GitCommit -> GitCommit -> Bool #
Hashable GitCommit Source #
Instance details Defined in Development.Benchmark.Rules Methods hashWithSalt :: Int -> GitCommit -> Int # hash :: GitCommit -> Int #
type Rep GitCommit Source #
Instance details Defined in Development.Benchmark.Rules type Rep GitCommit = D1 ('MetaData "GitCommit" "Development.Benchmark.Rules" "shake-bench-0.2.0.0-GBfwpyOoQh65kxC6vx003f" 'False) (C1 ('MetaCons "GitCommit" 'PrefixI 'True) ((S1 ('MetaSel ('Just "gitName") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text) :: S1 ('MetaSel ('Just "name") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text))) :: (S1 ('MetaSel ('Just "parent") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text)) :*: S1 ('MetaSel ('Just "include") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Bool))))