diff --git a/Makefile b/Makefile index c733300..9e54057 100755 --- a/Makefile +++ b/Makefile @@ -1,17 +1,22 @@ # # This makefile will build a small benchmarking utility for 'malloc' implementations and will -# run it with different implementations saving results into JSON files. +# run it with different implementations, first saving results into JSON files, and then plotting +# them graphically. # -# Specifically this makefile downloads, configure and compiles 3 different software packages: -# - GNU libc -# - Google perftools (tcmalloc) -# - jemalloc +# Specifically, this makefile downloads, configures and compiles these different software packages: +# 1. GNU libc +# 2. Google perftools (tcmalloc) +# 3. jemalloc # -# Tested with versions: -# - GNU libc 2.26 -# - Google perftools (tcmalloc) 2.6.3 -# - jemalloc 5.0.1 +# First tested with these versions: +# 1. GNU libc 2.26 +# 2. Google perftools (tcmalloc) 2.6.3 +# 3. jemalloc 5.0.1 # +# Most-recently tested on Ubuntu 20.04 with these versions: +# 1. GNU libc 2.31 +# 2. Google perftools (tcmalloc) 2.9.1 +# 3. jemalloc 5.2.1-742 # # @@ -76,6 +81,7 @@ glibc_url := git://sourceware.org/git/glibc.git tcmalloc_url := https://github.com/gperftools/gperftools.git jemalloc_url := https://github.com/jemalloc/jemalloc.git +# Alternate download version and source if not using the git repo above glibc_version := 2.26 glibc_alt_wget_url := https://ftpmirror.gnu.org/libc/glibc-$(glibc_version).tar.xz @@ -163,6 +169,7 @@ collect_results: @sudo lshw -short -class memory -class processor > $(results_dir)/hardware-inventory.txt @echo -n "Number of CPU cores: " >>$(results_dir)/hardware-inventory.txt @grep "processor" /proc/cpuinfo | wc -l >>$(results_dir)/hardware-inventory.txt + # NB: you may need to install `numactl` first with `sudo apt install numactl`. @(which numactl >/dev/null 2>&1) && echo "NUMA informations:" >>$(results_dir)/hardware-inventory.txt @(which numactl >/dev/null 2>&1) && numactl -H >>$(results_dir)/hardware-inventory.txt @@ -173,5 +180,5 @@ plot_results: upload_results: git add -f $(results_dir)/*$(benchmark_result_json) $(results_dir)/$(benchmark_result_png) $(results_dir)/hardware-inventory.txt git commit -m "Adding results from folder $(results_dir) to the GIT repository" - @echo "Run 'git push' to push online your results (required GIT repo write access)" + @echo "Run 'git push' to push online your results (requires GIT repo write access)" diff --git a/README.md b/README.md index 6636599..dc241dc 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,53 @@ +See the included glibc benchmark source code and readme here: [benchmark-src](benchmark-src) + + # malloc-benchmarks -Simple benchmarking scripts to run on any machine to compare different C/C++ malloc implementations. -The scripts are not meant to face any possible problem, quite the opposite. +Simple benchmarking and plotting scripts to run on any machine to compare different C/C++ malloc implementations. +These scripts are not meant to face any possible problem; quite the opposite. They will: - - download and build [GNU libc](https://www.gnu.org/software/libc/), [Google perftools](https://github.com/gperftools/gperftools), [Jemalloc](http://jemalloc.net/) - - use GNU libc malloc multi-thread benchmarking utility to generate JSON results for different combinations - of malloc implementation and number of threads - - use [Python matplotlib](https://matplotlib.org/) to produce a summary figure +1. Download and build [GNU libc](https://www.gnu.org/software/libc/), [Google perftools](https://github.com/gperftools/gperftools), [Jemalloc](http://jemalloc.net/) +1. Use the GNU libc malloc multi-threaded benchmarking utility to generate JSON results for different combinations + of malloc implementations and numbers of threads +1. Use [Python matplotlib](https://matplotlib.org/) to produce a plot of the results + + +## Dependencies + +If `make` below fails, you may need to install (via `sudo apt install`) one or more of the following. If you like, just begin by running the installation commands below. Last tested in Ubuntu 20.04. + +```bash +sudo apt update && sudo apt install \ + numactl g++ clang llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ + cmake python python3 ruby ninja-build libtool autoconf +# For Python +pip3 install matplotlib +``` ## How to collect benchmark results and view them +```bash +git clone https://github.com/f18m/malloc-benchmarks.git +cd malloc-benchmarks +make +# OR, time the process too to help you set expectations for how long it will take +time make +``` + +Once you have run `make`, the plot will display. To re-plot the results without rerunning the tests, run: +```bash +make plot_results ``` - git clone https://github.com/f18m/malloc-benchmarks.git - cd malloc-benchmarks - make + +Note that each time you run `make`, all of the benchmark results will be stored in a folder for your computer within the `results` dir, overwriting all previous results. So, if you wish to save previous benchmarking runs, be sure to rename your computer's folder in the `results` dir prior to running `make` again. + +You can customize the runs be setting environment variables as you call `make`. See the top of the `Makefile` for details. See the default values for `benchmark_nthreads` and `implem_list` in the `Makefile`. + +Examples: +```bash +# Run only 1 and 2 threads, testing only malloc implementations jemalloc and tcmalloc: +NTHREADS="1 2" IMPLEMENTATIONS="jemalloc tcmalloc" time make ``` @@ -22,28 +55,35 @@ They will: On the machine where you want to collect benchmark results: -``` - git clone https://github.com/f18m/malloc-benchmarks.git - cd malloc-benchmarks - make download build collect_results - scp -r results IP_OF_OTHER_MACHINE: +```bash +git clone https://github.com/f18m/malloc-benchmarks.git +cd malloc-benchmarks +make download build collect_results +scp -r results IP_OF_OTHER_MACHINE: ``` On the other machine where you want to plot results: -``` - git clone https://github.com/f18m/malloc-benchmarks.git - cd malloc-benchmarks - mv ../results . - make plot_results +```bash +git clone https://github.com/f18m/malloc-benchmarks.git +cd malloc-benchmarks +mv ../results . +make plot_results ``` ## Example benchmarks -The following are some pictures obtained on different HW systems using however the same benchmarking utility written by -GNU libc developers. They give an idea on how much performances can be different on different CPU/memory HW and varying the number of threads. -Of course the closer the curves are to zero, the better they are (the lower the better!). +The following are some plots of results obtained on different hardware systems using the same benchmarking utility written by the +GNU libc developers. They give an idea of how much performance can differ on different CPU/memory hardware and a varying the number of threads. +Of course, the closer the curves are to zero, the better they are (the lower the better!). + +**To verify the version numbers for your benchmarks, look in the following places after running `make`:** +1. **system_default:** run `apt show libc6` to see your system glibc version ([source: "Determining the Installed glibc Version"](https://www.linode.com/docs/guides/patching-glibc-for-the-ghost-vulnerability/)). Ex: `Version: 2.31-0ubuntu9.2` +1. **glibc:** See this file: `malloc-benchmarks/glibc/version.h` +1. **tcmalloc:** See the `TC_VERSION_STRING` value inside `malloc-benchmarks/tcmalloc-install/include/gperftools/tcmalloc.h` +1. **jemalloc:** See the `JEMALLOC_VERSION` value inside `malloc-benchmarks/jemalloc-install/include/jemalloc/jemalloc.h` +