diff --git a/Makefile b/Makefile index c733300..9e54057 100755 --- a/Makefile +++ b/Makefile @@ -1,17 +1,22 @@ # # This makefile will build a small benchmarking utility for 'malloc' implementations and will -# run it with different implementations saving results into JSON files. +# run it with different implementations, first saving results into JSON files, and then plotting +# them graphically. # -# Specifically this makefile downloads, configure and compiles 3 different software packages: -# - GNU libc -# - Google perftools (tcmalloc) -# - jemalloc +# Specifically, this makefile downloads, configures and compiles these different software packages: +# 1. GNU libc +# 2. Google perftools (tcmalloc) +# 3. jemalloc # -# Tested with versions: -# - GNU libc 2.26 -# - Google perftools (tcmalloc) 2.6.3 -# - jemalloc 5.0.1 +# First tested with these versions: +# 1. GNU libc 2.26 +# 2. Google perftools (tcmalloc) 2.6.3 +# 3. jemalloc 5.0.1 # +# Most-recently tested on Ubuntu 20.04 with these versions: +# 1. GNU libc 2.31 +# 2. Google perftools (tcmalloc) 2.9.1 +# 3. jemalloc 5.2.1-742 # # @@ -76,6 +81,7 @@ glibc_url := git://sourceware.org/git/glibc.git tcmalloc_url := https://github.com/gperftools/gperftools.git jemalloc_url := https://github.com/jemalloc/jemalloc.git +# Alternate download version and source if not using the git repo above glibc_version := 2.26 glibc_alt_wget_url := https://ftpmirror.gnu.org/libc/glibc-$(glibc_version).tar.xz @@ -163,6 +169,7 @@ collect_results: @sudo lshw -short -class memory -class processor > $(results_dir)/hardware-inventory.txt @echo -n "Number of CPU cores: " >>$(results_dir)/hardware-inventory.txt @grep "processor" /proc/cpuinfo | wc -l >>$(results_dir)/hardware-inventory.txt + # NB: you may need to install `numactl` first with `sudo apt install numactl`. @(which numactl >/dev/null 2>&1) && echo "NUMA informations:" >>$(results_dir)/hardware-inventory.txt @(which numactl >/dev/null 2>&1) && numactl -H >>$(results_dir)/hardware-inventory.txt @@ -173,5 +180,5 @@ plot_results: upload_results: git add -f $(results_dir)/*$(benchmark_result_json) $(results_dir)/$(benchmark_result_png) $(results_dir)/hardware-inventory.txt git commit -m "Adding results from folder $(results_dir) to the GIT repository" - @echo "Run 'git push' to push online your results (required GIT repo write access)" + @echo "Run 'git push' to push online your results (requires GIT repo write access)" diff --git a/README.md b/README.md index 6636599..dc241dc 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,53 @@ +See the included glibc benchmark source code and readme here: [benchmark-src](benchmark-src) + + # malloc-benchmarks -Simple benchmarking scripts to run on any machine to compare different C/C++ malloc implementations. -The scripts are not meant to face any possible problem, quite the opposite. +Simple benchmarking and plotting scripts to run on any machine to compare different C/C++ malloc implementations. +These scripts are not meant to face any possible problem; quite the opposite. They will: - - download and build [GNU libc](https://www.gnu.org/software/libc/), [Google perftools](https://github.com/gperftools/gperftools), [Jemalloc](http://jemalloc.net/) - - use GNU libc malloc multi-thread benchmarking utility to generate JSON results for different combinations - of malloc implementation and number of threads - - use [Python matplotlib](https://matplotlib.org/) to produce a summary figure +1. Download and build [GNU libc](https://www.gnu.org/software/libc/), [Google perftools](https://github.com/gperftools/gperftools), [Jemalloc](http://jemalloc.net/) +1. Use the GNU libc malloc multi-threaded benchmarking utility to generate JSON results for different combinations + of malloc implementations and numbers of threads +1. Use [Python matplotlib](https://matplotlib.org/) to produce a plot of the results + + +## Dependencies + +If `make` below fails, you may need to install (via `sudo apt install`) one or more of the following. If you like, just begin by running the installation commands below. Last tested in Ubuntu 20.04. + +```bash +sudo apt update && sudo apt install \ + numactl g++ clang llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ + cmake python python3 ruby ninja-build libtool autoconf +# For Python +pip3 install matplotlib +``` ## How to collect benchmark results and view them +```bash +git clone https://github.com/f18m/malloc-benchmarks.git +cd malloc-benchmarks +make +# OR, time the process too to help you set expectations for how long it will take +time make +``` + +Once you have run `make`, the plot will display. To re-plot the results without rerunning the tests, run: +```bash +make plot_results ``` - git clone https://github.com/f18m/malloc-benchmarks.git - cd malloc-benchmarks - make + +Note that each time you run `make`, all of the benchmark results will be stored in a folder for your computer within the `results` dir, overwriting all previous results. So, if you wish to save previous benchmarking runs, be sure to rename your computer's folder in the `results` dir prior to running `make` again. + +You can customize the runs be setting environment variables as you call `make`. See the top of the `Makefile` for details. See the default values for `benchmark_nthreads` and `implem_list` in the `Makefile`. + +Examples: +```bash +# Run only 1 and 2 threads, testing only malloc implementations jemalloc and tcmalloc: +NTHREADS="1 2" IMPLEMENTATIONS="jemalloc tcmalloc" time make ``` @@ -22,28 +55,35 @@ They will: On the machine where you want to collect benchmark results: -``` - git clone https://github.com/f18m/malloc-benchmarks.git - cd malloc-benchmarks - make download build collect_results - scp -r results IP_OF_OTHER_MACHINE: +```bash +git clone https://github.com/f18m/malloc-benchmarks.git +cd malloc-benchmarks +make download build collect_results +scp -r results IP_OF_OTHER_MACHINE: ``` On the other machine where you want to plot results: -``` - git clone https://github.com/f18m/malloc-benchmarks.git - cd malloc-benchmarks - mv ../results . - make plot_results +```bash +git clone https://github.com/f18m/malloc-benchmarks.git +cd malloc-benchmarks +mv ../results . +make plot_results ``` ## Example benchmarks -The following are some pictures obtained on different HW systems using however the same benchmarking utility written by -GNU libc developers. They give an idea on how much performances can be different on different CPU/memory HW and varying the number of threads. -Of course the closer the curves are to zero, the better they are (the lower the better!). +The following are some plots of results obtained on different hardware systems using the same benchmarking utility written by the +GNU libc developers. They give an idea of how much performance can differ on different CPU/memory hardware and a varying the number of threads. +Of course, the closer the curves are to zero, the better they are (the lower the better!). + +**To verify the version numbers for your benchmarks, look in the following places after running `make`:** +1. **system_default:** run `apt show libc6` to see your system glibc version ([source: "Determining the Installed glibc Version"](https://www.linode.com/docs/guides/patching-glibc-for-the-ghost-vulnerability/)). Ex: `Version: 2.31-0ubuntu9.2` +1. **glibc:** See this file: `malloc-benchmarks/glibc/version.h` +1. **tcmalloc:** See the `TC_VERSION_STRING` value inside `malloc-benchmarks/tcmalloc-install/include/gperftools/tcmalloc.h` +1. **jemalloc:** See the `JEMALLOC_VERSION` value inside `malloc-benchmarks/jemalloc-install/include/jemalloc/jemalloc.h` + diff --git a/bench_collect_results.py b/bench_collect_results.py index d6fd9f3..2950cee 100755 --- a/bench_collect_results.py +++ b/bench_collect_results.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """Generate benchmarking results in JSON form, using GNU libc benchmarking utility; different allocators are injected into that utility by using LD_PRELOAD trick. """ @@ -43,7 +43,7 @@ def find(name, paths): for path in paths: - #print "Searching into: ", path + #print("Searching into: ", path) for root, dirs, files in os.walk(path, followlinks=False): if name in files: return os.path.join(root, name) @@ -107,7 +107,7 @@ def run_benchmark(outfile,thread_values,impl_name): os.environ["LD_PRELOAD"] = impl_preload_libs[impl_name] if len(os.environ["LD_PRELOAD"])>0: # the tcmalloc/jemalloc shared libs require in turn C++ libs: - #print "preload_required_libs_fullpaths is:", preload_required_libs_fullpaths + #print("preload_required_libs_fullpaths is:", preload_required_libs_fullpaths) for lib in preload_required_libs_fullpaths: os.environ["LD_PRELOAD"] = os.environ["LD_PRELOAD"] + ':' + lib @@ -156,13 +156,13 @@ def main(args): sys.exit(3) outfile = os.path.join(outfile_path_prefix, implementations[idx] + '-' + outfile_postfix) - print "----------------------------------------------------------------------------------------------" - print "Testing implementation '{}'. Saving results into '{}'".format(implementations[idx],outfile) + print("----------------------------------------------------------------------------------------------") + print("Testing implementation '{}'. Saving results into '{}'".format(implementations[idx],outfile)) - print "Will run tests for {} different number of threads".format(len(thread_values)) + print("Will run tests for {} different number of threads".format(len(thread_values))) success = success + run_benchmark(outfile,thread_values,implementations[idx]) - print "----------------------------------------------------------------------------------------------" + print("----------------------------------------------------------------------------------------------") return success if __name__ == '__main__': diff --git a/bench_plot_results.py b/bench_plot_results.py index 22e3ef9..d1b71a8 100755 --- a/bench_plot_results.py +++ b/bench_plot_results.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """Generates a figure that shows all benchmarking results """ import sys @@ -8,7 +8,7 @@ import matplotlib.pyplot as plotlib -BenchmarkPoint = collections.namedtuple('BenchmarkPoint', ['threads', 'time_per_iteration'], verbose=False) +BenchmarkPoint = collections.namedtuple('BenchmarkPoint', ['threads', 'time_per_iteration']) filled_markers = ('o', 'v', '^', '<', '>', '8', 's', 'p', '*', 'h', 'H', 'D', 'd', 'P', 'X') colours = ('r', 'g', 'b', 'black', 'yellow', 'purple') @@ -32,8 +32,12 @@ def plot_graphs(outfilename, benchmark_dict): plotlib.setp(lines, 'color', colours[nmarker]) # remember max X/Y - max_x.append(max(X)) - max_y.append(max(Y)) + # In case you only ran some of the tests, don't attempt to get `max()` on an empty list--ie: + # for a benchmark you didn't run. Only operate if the lists aren't empty. + if X: + max_x.append(max(X)) + if Y: + max_y.append(max(Y)) nmarker=nmarker+1 @@ -42,6 +46,12 @@ def plot_graphs(outfilename, benchmark_dict): plotlib.ylim(0, max(max_y)*1.3) print("Writing plot into '%s'" % outfilename) + print("- - -\n" + + "Close the plot to terminate the program. Run `make plot_results` to plot the results\n" + + "again. Be sure to manually make a copy of the \"results\" folder (if you wish to\n" + + "save your results) before running `make` again, or else these results will be\n" + + "overwritten.\n" + + "- - -") plotlib.legend(loc='upper left') plotlib.savefig(outfilename) plotlib.show() diff --git a/benchmark-src/README.md b/benchmark-src/README.md index 61a538b..7033b55 100644 --- a/benchmark-src/README.md +++ b/benchmark-src/README.md @@ -1,4 +1,21 @@ # Benchmark utility sources -The sources collected here are a small subset of GNU libc benchmarking utility for 'malloc' implementations. -Please see glibc/benchtests/bech-malloc-thread.c for the most up-to-date source code. +The sources collected here are a small subset of the GNU libc benchmarking utility for `malloc` implementations. + +Please see `glibc/benchtests/bech-malloc-thread.c` for the most up-to-date source code. + +You can obtain it here: https://www.gnu.org/software/libc/sources.html: +```bash +git clone https://sourceware.org/git/glibc.git +cd glibc +git checkout master +``` + +Mirror: https://github.com/bminor/glibc/tree/master/benchtests + +See also: +1. https://kazoo.ga/a-simple-tool-to-test-malloc-performance/ + + +## Each of the above source files came from the following glibc paths: +1. todo