Allow perf harness to compare multiple revisions of SwiftProtobuf (apple#426)

* Separate perf harness generator and runner scripts.
* Update perf harness to compare revisions.
* Remove commented code; add checkout cleanup hook.

Author: allevato

Performance/css/harness-visualization.css

@@ -16,7 +16,7 @@ table.numeric td {
 
 table.numeric th {
   background-color: #eee;
-  text-align: right;
+  text-align: center;
 }
 
 table.numeric tfoot td {

Performance/perf_runner_cpp.sh renamed to Performance/generators/cpp.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# SwiftProtobuf/Performance/perf_runner_cpp.sh - C++ test harness generator
+# SwiftProtobuf/Performance/generators/cpp.sh - C++ test harness generator
 #
 # This source file is part of the Swift.org open source project
 #
@@ -16,8 +16,6 @@
 #
 # -----------------------------------------------------------------------------
 
-set -eu
-
 function print_cpp_set_field() {
   num=$1
   type=$2
@@ -157,32 +155,3 @@ EOF
 }
 EOF
 }
-
-function run_cpp_harness() {
-  harness="$1"
-
-  echo "Generating C++ harness source..."
-  gen_harness_path="$script_dir/_generated/Harness+Generated.cc"
-  generate_cpp_harness "$field_count" "$field_type"
-
-  echo "Building C++ test harness..."
-  time ( g++ --std=c++11 -O \
-      -o "$harness" \
-      -I "$script_dir" \
-      -I "$GOOGLE_PROTOBUF_CHECKOUT/src" \
-      -L "$GOOGLE_PROTOBUF_CHECKOUT/src/.libs" \
-      -lprotobuf \
-      "$gen_harness_path" \
-      "$script_dir/Harness.cc" \
-      "$script_dir/_generated/message.pb.cc" \
-      "$script_dir/main.cc" \
-  )
-  echo
-
-  # Make sure the dylib is loadable from the harness if the user hasn't
-  # actually installed them.
-  cp "$GOOGLE_PROTOBUF_CHECKOUT"/src/.libs/libprotobuf.*.dylib \
-      "$script_dir/_generated"
-
-  run_harness_and_concatenate_results "C++" "$harness" "$partial_results"
-}

Performance/perf_runner_swift.sh renamed to Performance/generators/swift.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# SwiftProtobuf/Performance/perf_runner_swift.sh - Swift test harness generator
+# SwiftProtobuf/Performance/generators/swift.sh - Swift test harness generator
 #
 # This source file is part of the Swift.org open source project
 #
@@ -16,8 +16,6 @@
 #
 # -----------------------------------------------------------------------------
 
-set -eu
-
 function print_swift_set_field() {
   num=$1
   type=$2
@@ -86,8 +84,8 @@ extension Harness {
       }
 
       // Exercise text serialization.
-      let text = try measureSubtask("Encode text") {
-        return try message.textFormatString()
+      let text = measureSubtask("Encode text") {
+        return message.textFormatString()
       }
       _ = try measureSubtask("Decode text") {
         return try PerfMessage(textFormatString: text)
@@ -114,41 +112,3 @@ EOF
 }
 EOF
 }
-
-function run_swift_harness() {
-  harness="$1"
-
-  echo "Generating Swift harness source..."
-  gen_harness_path="$script_dir/_generated/Harness+Generated.swift"
-  generate_swift_harness "$field_count" "$field_type"
-
-  # Build the dynamic library to use in the tests from the .o files that were
-  # already built for the plug-in.
-  echo "Building SwiftProtobuf dynamic library..."
-  swiftc -emit-library -O -wmo -target x86_64-apple-macosx10.10 \
-      -o "$script_dir/_generated/libSwiftProtobuf.dylib" \
-      "$script_dir/../Sources/SwiftProtobuf/"*.swift
-
-  echo "Building Swift test harness..."
-  time ( swiftc -O -target x86_64-apple-macosx10.10 \
-      -o "$harness" \
-      -I "$script_dir/../.build/release" \
-      -L "$script_dir/_generated" \
-      -lSwiftProtobuf \
-      "$gen_harness_path" \
-      "$script_dir/Harness.swift" \
-      "$script_dir/_generated/message.pb.swift" \
-      "$script_dir/main.swift"
-  )
-  echo
-
-  dylib="$script_dir/_generated/libSwiftProtobuf.dylib"
-  echo "Swift dylib size before stripping: $(stat -f "%z" "$dylib") bytes"
-  cp "$dylib" "${dylib}_stripped"
-  strip -u -r "${dylib}_stripped"
-  echo "Swift dylib size after stripping:  $(stat -f "%z" "${dylib}_stripped") bytes"
-  echo
-
-  run_harness_and_concatenate_results "Swift" "$harness_swift" "$partial_results"
-  profile_harness "Swift" "$harness_swift"
-}

Performance/js/harness-visualization.js

@@ -9,10 +9,6 @@
     'Equality',
   ];
 
-  // The languages we have harnesses for. The results for each language will
-  // appear as a series in the plot.
-  var languages = ['Swift', 'C++'];
-
   // The harnessSize keys we want to print in the summary table, in the order
   // they should be displayed.
   var harnessSizeKeys = ['Unstripped', 'Stripped'];
@@ -50,7 +46,7 @@
   };
 
   // Creates and return a series for the given language's results in a session.
-  function createSeries(session, language) {
+  function createSeries(session, series) {
     var x = [];
     var y = [];
 
@@ -59,7 +55,7 @@
     // vertical, which is what we want.
     for (var i = 0; i < benchmarks.length; i++) {
       var benchmark = benchmarks[i];
-      var timings = session[language][benchmark];
+      var timings = series.data[benchmark];
       if (timings) {
         for (var j = 0; j < timings.length; j++) {
           x.push(benchmark.replace(" ", "<br>"));
@@ -69,7 +65,7 @@
     }
 
     return {
-      name: language,
+      name: series.name,
       x: x,
       y: y,
       type: 'box',
@@ -126,10 +122,8 @@
     // Insert the runtime stats.
     var header = $('<tr></tr>').appendTo(table);
     header.append($('<th>Median runtimes</th>'));
-    for (var j = 0; j < languages.length; j++) {
-      header.append($('<th></th>').text(languages[j]));
-      header.append($('<th></th>'));
-      header.append($('<th></th>'));
+    for (var j = 0; j < session.series.length; j++) {
+      header.append($('<th colspan="2"></th>').text(session.series[j].name));
     }
 
     for (var i = 0; i < benchmarks.length; i++) {
@@ -142,9 +136,8 @@
       // Track which language was the fastest
       var timings = [];
       var bestLanguage = 0;
-      for (var j = 0; j < languages.length; j++) {
-        var language = languages[j];
-        var languageTimings = session[language][benchmark];
+      for (var j = 0; j < session.series.length; j++) {
+        var languageTimings = session.series[j].data[benchmark];
         if (languageTimings) {
           var med = median(languageTimings);
           timings.push(med);
@@ -156,8 +149,7 @@
 
       // Insert the per-language timings into the table
       var bestValue = timings[bestLanguage];
-      for (var j = 0; j < languages.length; j++) {
-        var language = languages[j];
+      for (var j = 0; j < session.series.length; j++) {
         var med = timings[j];
         var valueCell = $('<td></td>').appendTo(tr);
         var multiplierCell = $('<td></td>').appendTo(tr);
@@ -171,15 +163,13 @@
           }
           decorateMultiplierCell(multiplierCell, multiplier);
         }
-        tr.append($('<td></td>'));
       }
     }
 
     // Insert the binary size stats.
     header = $('<tr></tr>').appendTo(table);
     header.append($('<th>Harness size</th>'));
-    for (var j = 0; j < languages.length; j++) {
-      header.append($('<th></th>'));
+    for (var j = 0; j < session.series.length; j++) {
       header.append($('<th></th>'));
       header.append($('<th></th>'));
     }
@@ -192,16 +182,15 @@
 
       var bestLanguage = 0;
       var sizes = [];
-      for (var j = 0; j < languages.length; j++) {
-        var language = languages[j];
-          var size = session[language].harnessSize[harnessSizeKey];
+      for (var j = 0; j < session.series.length; j++) {
+          var size = session.series[j].data.harnessSize[harnessSizeKey];
           sizes.push(size);
           if (size < sizes[bestLanguage]) {
               bestLanguage = j;
           }
       }
 
-      for (var j = 0; j < languages.length; j++) {
+      for (var j = 0; j < session.series.length; j++) {
         var size = sizes[j];
         var multiplier = size / sizes[bestLanguage];
         var formattedSize = size.toLocaleString() + '&nbsp;b';
@@ -213,13 +202,12 @@
             multiplierCell.text('(' + multiplier.toFixed(1) + 'x)');
         }
         decorateMultiplierCell(multiplierCell, multiplier);
-        tr.append($('<td></td>'));
       }
     }
 
     var tfoot = $('<tfoot></tfoot>').appendTo(table);
     var footerRow = $('<tr></tr>').appendTo(tfoot);
-    var colspan = 3 * languages.length + 1;
+    var colspan = 3 * session.series.length + 1;
     var footerCell =
         $('<td colspan="' + colspan + '"></td>').appendTo(footerRow);
       footerCell.text('Green highlights the best result for each test. ' +
@@ -247,21 +235,21 @@
       var title = session.type;
       var header = $('<h3></h3>').addClass('row').text(title);
 
-      var subtitle = 'Branch <tt>' + session.branch +
-          '</tt>, commit <tt>' + session.commit + '</tt>';
+      var subtitle = 'Working tree was at <tt>' + session.branch +
+          '</tt>, commit <tt>' + session.commit.substr(0, 6) + '</tt>';
       if (session.uncommitted_changes) {
         subtitle += ' (with uncommited changes)';
       }
-      subtitle += ', run on ' + formattedDate;
+      subtitle += ' &ndash; ' + formattedDate;
 
       header.append($('<small></small>').html(subtitle));
       $('#container').append('<hr>');
       $('#container').append(header);
 
       var id = 'chart' + i;
       var row = $('<div></div>').addClass('row');
-      var chartColumn = $('<div></div>').addClass('col-md-9');
-      var tableColumn = $('<div></div>').addClass('col-md-3');
+      var chartColumn = $('<div></div>').addClass('col-md-8');
+      var tableColumn = $('<div></div>').addClass('col-md-4');
 
       row.append(chartColumn);
       row.append(tableColumn);
@@ -270,12 +258,9 @@
       var chart = $('<div></div>').attr('id', id).addClass('chart');
       chartColumn.append(chart);
 
-      for (var j = 0; j < languages.length; j++) {
-        var language = languages[j];
-        if (session[language]) {
-          var series = createSeries(session, language);
-          allSeries.push(series);
-        }
+      for (var j = 0; j < session.series.length; j++) {
+        var series = createSeries(session, session.series[j]);
+        allSeries.push(series);
       }
 
       Plotly.newPlot(id, allSeries, layout, {