separate platform/benchmark results

- separate results by platforms and benchmarks
- correctly generate `table.html` with missing results
- use geometric mean and standard deviation

Author: daiyam

webdriver-java/makeTable.js

@@ -1,67 +1,89 @@
 "use strict";
 
-var _ = require('lodash');
-var fs = require('fs');
-
-var files = fs.readdirSync("./results")
-	.filter(file => file.endsWith('.txt'));
+const _ = require('lodash');
+const dots = require('dot').process({
+	path: './'
+});
+const fs = require('fs');
 
-let results = new Map();
+let results = {};
 let frameworks = [];
+let benchmarks = [];
 
-files.map(f => {return {name: f, data: fs.readFileSync("./results/"+f, {encoding:"utf-8"})};})
-	.forEach(({name, data}) => {
-		let n = name.substring(0,name.length-4);
-		results.set(n, Function("return "+data)());
-		frameworks.push(n);
-	});
+fs.readdirSync('./results').filter(file => file.endsWith('.json')).forEach(name => {
+	let data = JSON.parse(fs.readFileSync('./results/' + name, {
+		encoding:'utf-8'
+	}));
+	
+	frameworks.push(data.framework);
+	benchmarks.push(data.benchmark);
+	
+	results[data.framework] = results[data.framework] || {};
+	results[data.framework][data.benchmark] = data;
+});
 
-let benchmarks = results.values().next().value.results.map(r => r.benchmark);
+frameworks = _.uniq(frameworks);
+benchmarks = _.uniq(benchmarks);
 
-let getValue = (framework, benchmark) => results.get(framework).results.find(r => r.benchmark === benchmark).avg;
+let getValue = (framework, benchmark) => results[framework] && results[framework][benchmark];
 
-var dots = require('dot').process({
-	path: './'
-});
-
-var factors = frameworks.map(f => 1.0);
+let factors = frameworks.map(f => 1.0);
 
-var benches = [];
+let benches = [];
 benchmarks.forEach(benchmark => {
-	var bench = {
+	let bench = {
 		name: benchmark,
 		tests: []
 	};
 
-	var values = [];
+	let values = [];
 	frameworks.forEach(framework => {
 		values.push(getValue(framework, benchmark));
 	});
-	var sorted = values.slice(0).sort(function(a, b) {
-		return a - b;
-	});
-	var min = sorted[0];
-	var max = sorted[sorted.length - 1];
 
-	var top1 = min * 1.33;
-	var top3 = min * 2.33;
-
-	while(top1 >= max) {
-		top1 *= 0.8;
-		top3 *= 0.8;
-	}
-	while(top3 >= max) {
-		top3 *= 0.8;
+	let sorted = _.compact(values).map(data => {
+		return data.geometricMean;
+	}).sort((a, b) => a - b);
+	
+	let min, top1, top3;
+	
+	if(sorted.length) {
+		min = sorted[0];
+		
+		if(sorted.length === 1) {
+			top1 = min;
+		}
+		else {
+			let min = sorted[1];
+			let max = sorted[sorted.length - 1];
+			
+			top1 = min * 1.33;
+			top3 = min * 2.33;
+		
+			while(top1 >= max) {
+				top1 *= 0.8;
+				top3 *= 0.8;
+			}
+			while(top3 >= max) {
+				top3 *= 0.8;
+			}
+		}
 	}
 
 	_.forEach(values, function(value, idx) {
-		let factor = Math.max(16,value)/Math.max(16,min);
-		factors[idx] = factors[idx] * factor;
-		bench.tests.push({
-			value,
-			factor : factor.toPrecision(3),
-			class: value <= top1 ? 'top1' : value <= top3 ? 'top3' : 'top5'
-		});
+		if(value) {
+			let factor = Math.max(16,value.geometricMean)/Math.max(16,min);
+			factors[idx] = factors[idx] * factor;
+			bench.tests.push({
+				mean: value.geometricMean,
+				deviation: value.standardDeviation,
+				factor : factor.toPrecision(3),
+				class: value.geometricMean <= top1 ? 'top1' : value.geometricMean <= top3 ? 'top3' : 'top5'
+			});
+		}
+		else {
+			bench.tests.push(null);
+		}
 	})
 	benches.push(bench);
 });
@@ -72,7 +94,7 @@ let geomMeans = factors.map(f => {
 });
 
 fs.writeFileSync('./table.html', dots.table({
-	frameworks: [''].concat(frameworks).map(framework => framework.replace('-v', ' v').replace('/dist', '')),
+	frameworks: [''].concat(frameworks).map(framework => framework.replace('-v', ' v')),
 	benches,
 	geomMeans
 }), {

webdriver-java/pom.xml

@@ -38,6 +38,11 @@
             <artifactId>guava</artifactId>
             <version>19.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-math3</artifactId>
+            <version>3.6.1</version>
+        </dependency>
     </dependencies>
 
     <build>