|
#!/usr/bin/env ruby |
|
# coding: utf-8 |
|
|
|
require 'getoptlong' |
|
require 'pathname' |
|
|
|
$benchmarks_all = [ |
|
# Single-threaded benchmarks. |
|
"churn", |
|
"list_allocate", |
|
"tree_allocate", |
|
"tree_churn", |
|
"fragment", |
|
"fragment_iterate", |
|
"medium", |
|
"big", |
|
|
|
# Benchmarks based on browser recordings. |
|
"facebook", |
|
"reddit", |
|
"flickr", |
|
"theverge", |
|
"nimlang", |
|
|
|
# Multi-threaded benchmark variants. |
|
"message_one", |
|
"message_many", |
|
"churn --parallel", |
|
"list_allocate --parallel", |
|
"tree_allocate --parallel", |
|
"tree_churn --parallel", |
|
"fragment --parallel", |
|
"fragment_iterate --parallel", |
|
|
|
# These tests often crash TCMalloc: <rdar://problem/13657137>. |
|
"medium --parallel", |
|
"big --parallel", |
|
|
|
# Enable these tests to test memory footprint. The way they run is not |
|
# really compatible with throughput testing. |
|
# "reddit_memory_warning --runs 0", |
|
# "flickr_memory_warning --runs 0", |
|
# "theverge_memory_warning --runs 0", |
|
|
|
# Enable this test to test shrinking back down from a large heap while a process remains active. |
|
# The way it runs is not really compatible with throughput testing. |
|
# "balloon" |
|
"facebook --parallel", |
|
"reddit --parallel", |
|
"flickr --parallel", |
|
"theverge --parallel", |
|
# "nimlang --use-thread-id", |
|
] |
|
|
|
$benchmarks_memory = [ |
|
"facebook", |
|
"reddit", |
|
"flickr", |
|
"theverge", |
|
"nimlang" |
|
] |
|
|
|
$benchmarks_memory_warning = [ |
|
"reddit_memory_warning --runs 0", |
|
"flickr_memory_warning --runs 0", |
|
"theverge_memory_warning --runs 0", |
|
] |
|
|
|
$benchmarks = $benchmarks_all |
|
$heap = 0 |
|
|
|
def usage |
|
puts "run-malloc-benchmarks [options] /path/to/MallocBench Name:/path/to/libmbmalloc.dylib [ Name:/path/to/libmbmalloc.dylib ]" |
|
puts |
|
puts " Runs a suite of memory allocation and access benchmarks." |
|
puts |
|
puts " <Name:/path/to/libmbmalloc.dylib> is a symbolic name followed by a path to libmbmalloc.dylib." |
|
puts |
|
puts " Specify \"SystemMalloc\" to test the built-in libc malloc." |
|
puts " Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone." |
|
puts |
|
puts " Example usage:" |
|
puts |
|
puts " run-malloc-benchmarks /BUILD/MallocBench SystemMalloc:/BUILD/libmbmalloc.dylib NanoMalloc:/BUILD/libmbmalloc.dylib" |
|
puts " run-malloc-benchmarks /BUILD/MallocBench FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib" |
|
puts " run-malloc-benchmarks --benchmark churn SystemMalloc:/BUILD/libmbmalloc.dylib FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib" |
|
puts |
|
puts "Options:" |
|
puts |
|
puts " --benchmark <benchmark> Select a single benchmark to run instead of the full suite." |
|
puts " --heap <heap> Set a baseline heap size." |
|
puts |
|
end |
|
|
|
class Dylib |
|
attr_reader :name |
|
attr_reader :path |
|
|
|
def initialize(name, path) |
|
@name = name |
|
@path = path |
|
end |
|
end |
|
|
|
class Results |
|
attr_reader :executionTime |
|
attr_reader :peakMemory |
|
attr_reader :memoryAtEnd |
|
|
|
def initialize(executionTime, peakMemory, memoryAtEnd) |
|
@executionTime = executionTime |
|
@peakMemory = peakMemory |
|
@memoryAtEnd = memoryAtEnd |
|
end |
|
end |
|
|
|
class Stat |
|
attr_reader :benchmark |
|
attr_reader :result |
|
|
|
def initialize(benchmark, result) |
|
@benchmark = benchmark |
|
@result = result[/\d+/].to_i |
|
end |
|
end |
|
|
|
class TimeStat < Stat |
|
def to_s |
|
@result + "ms" |
|
end |
|
end |
|
|
|
class MemoryStat < Stat |
|
def to_s |
|
@result + "kB" |
|
end |
|
end |
|
|
|
class PeakMemoryStat < Stat |
|
def to_s |
|
@result + "kB" |
|
end |
|
end |
|
|
|
def lpad(str, chars) |
|
if str.length > chars |
|
str |
|
else |
|
"%#{chars}s"%(str) |
|
end |
|
end |
|
|
|
def rpad(str, chars) |
|
while str.length < chars |
|
str += " " |
|
end |
|
str |
|
end |
|
|
|
def computeArithmeticMean(array) |
|
sum = 0.0 |
|
array.each { |
|
| value | |
|
sum += value |
|
} |
|
(sum / array.length) |
|
end |
|
|
|
def computeGeometricMean(array) |
|
mult = 1.0 |
|
array.each { |
|
| value | |
|
mult *= value ? value : 1.0 |
|
} |
|
(mult ** (1.0 / array.length)) |
|
end |
|
|
|
def computeHarmonicMean(array) |
|
1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value }) |
|
end |
|
|
|
def lowerIsBetter(a, b, better, worse) |
|
if b < a |
|
return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better |
|
end |
|
|
|
if b == a |
|
return "" |
|
end |
|
|
|
"! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse |
|
end |
|
|
|
|
|
def lowerIsFaster(a, b) |
|
lowerIsBetter(a, b, "faster", "slower") |
|
end |
|
|
|
def lowerIsSmaller(a, b) |
|
lowerIsBetter(a, b, "smaller", "bigger") |
|
end |
|
|
|
def numberWithDelimiter(number) |
|
number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse |
|
end |
|
|
|
def prettify(number, suffix) |
|
numberWithDelimiter(number) + suffix |
|
end |
|
|
|
def parseOptions |
|
GetoptLong.new( |
|
['--benchmark', GetoptLong::REQUIRED_ARGUMENT], |
|
['--memory', GetoptLong::NO_ARGUMENT], |
|
['--memory_warning', GetoptLong::NO_ARGUMENT], |
|
['--heap', GetoptLong::REQUIRED_ARGUMENT], |
|
['--help', GetoptLong::NO_ARGUMENT], |
|
).each { |
|
| opt, arg | |
|
case opt |
|
when '--benchmark' |
|
$benchmarks = [ arg ] |
|
when '--memory' |
|
$benchmarks = $benchmarks_memory |
|
when '--memory_warning' |
|
$benchmarks = $benchmarks_memory_warning |
|
when '--heap' |
|
$heap = arg |
|
when '--help' |
|
usage |
|
exit 1 |
|
else |
|
raise "bad option: #{opt}" |
|
end |
|
} |
|
|
|
if ARGV.length < 1 |
|
puts "Error: No MallocBench specified." |
|
exit 1 |
|
end |
|
|
|
if ARGV.length < 2 |
|
puts "Error: No dylib specified." |
|
exit 1 |
|
end |
|
|
|
$mallocBench = File.absolute_path(ARGV.shift) |
|
if !File.exists?($mallocBench) |
|
puts "File not found: #{$mallocBench}." |
|
exit 1 |
|
end |
|
|
|
$buildDir = Pathname.new($mallocBench).dirname |
|
|
|
dylibs = [] |
|
ARGV.each { |
|
| arg | |
|
name, path = arg.split(":") |
|
if !name || name.length < 1 || |
|
!path || path.length < 1 |
|
puts "Invalid <Name:/path/to/dylib>: '#{arg}'." |
|
exit 1 |
|
end |
|
|
|
dylib = Dylib.new(name, File.expand_path(path)) |
|
|
|
if !File.exists?(dylib.path) |
|
puts "File not found: #{dylib.path}." |
|
exit 1 |
|
end |
|
|
|
dylibs.push(dylib) |
|
} |
|
dylibs |
|
end |
|
|
|
def runBenchmarks(dylibs) |
|
executionTime = [] |
|
peakMemory = [] |
|
memoryAtEnd = [] |
|
|
|
$benchmarks.each { |
|
| benchmark | |
|
|
|
executionTime.push([]) |
|
peakMemory.push([]) |
|
memoryAtEnd.push([]) |
|
|
|
dylibs.each { |
|
| dylib | |
|
|
|
$stderr.print "\rRUNNING #{dylib.name}: #{benchmark}... " |
|
env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' " |
|
env += "LD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' " |
|
if dylib.name == "NanoMalloc" |
|
env += "MallocNanoZone=1 " |
|
elsif dylib.name == "SystemMalloc" |
|
env += "MallocNanoZone=0 " |
|
end |
|
input = "cd '#{$buildDir}'; #{env} '#{$mallocBench}' --benchmark #{benchmark} --heap #{$heap}}" |
|
output =`#{input}` |
|
splitOutput = output.split("\n") |
|
|
|
executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1])) |
|
peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0")) |
|
memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0")) |
|
} |
|
} |
|
$stderr.print "\r \n" |
|
|
|
Results.new(executionTime, peakMemory, memoryAtEnd) |
|
end |
|
|
|
def printResults(dylibs, results) |
|
def printHeader(dylibs, fieldSize) |
|
print |
|
print lpad("", fieldSize) |
|
print lpad(dylibs[0].name, fieldSize) |
|
if dylibs.length > 1 |
|
print lpad(dylibs[1].name, fieldSize) |
|
print lpad("Δ", fieldSize) |
|
end |
|
print "\n" |
|
end |
|
|
|
def printMetric(name, results, compareFunction, suffix, fieldSize) |
|
def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize) |
|
means = [] |
|
|
|
means.push(meanFunction.call(results.collect { | stats | stats[0].result })) |
|
print rpad(" " + name, fieldSize) |
|
print lpad("#{prettify(means[0].round, suffix)}", fieldSize) |
|
|
|
if results[0][1] |
|
means.push(meanFunction.call(results.collect { | stats | stats[1].result })) |
|
print lpad("#{prettify(means[1].round, suffix)}", fieldSize) |
|
print lpad(compareFunction.call(means[0], means[1]), fieldSize) |
|
end |
|
|
|
print "\n" |
|
end |
|
|
|
if results[0][0].result == 0 |
|
return |
|
end |
|
|
|
print name + ":\n" |
|
results.each { |
|
| stats | |
|
|
|
print rpad(" " + stats[0].benchmark, fieldSize) |
|
print lpad("#{prettify(stats[0].result, suffix)}", fieldSize) |
|
|
|
if stats[1] |
|
print lpad("#{prettify(stats[1].result, suffix)}", fieldSize) |
|
print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize) |
|
end |
|
|
|
print "\n" |
|
} |
|
|
|
print "\n" |
|
|
|
printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize) |
|
printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize) |
|
printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize) |
|
|
|
print "\n" |
|
end |
|
|
|
fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect { |
|
| benchmark | |
|
benchmark.size |
|
}.max + 4 |
|
|
|
printHeader(dylibs, fieldSize) |
|
printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize) |
|
printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize) |
|
printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize) |
|
end |
|
|
|
def main |
|
begin |
|
dylibs = parseOptions() |
|
results = runBenchmarks(dylibs) |
|
printResults(dylibs, results) |
|
rescue => exception |
|
puts |
|
puts |
|
puts exception |
|
puts exception.backtrace |
|
puts |
|
end |
|
end |
|
|
|
main() |
