update 20181028-01:

.gitignore

@@ -0,0 +1,2 @@
+.idea
+*.pyc

Dockerfile

@@ -1,8 +1,24 @@
-FROM debian
+FROM debian:9.5
 MAINTAINER Peter Naudus <uselinux@gmail.com>
 RUN apt-get -y update
-RUN apt-get -y install python python-dev python-pip apache2 apache2-dev libev-dev gcc
-RUN pip install cherrypy tornado uwsgi gunicorn bjoern meinheld mod_wsgi
+RUN apt-get -y  --no-install-recommends install python python-dev python-pip apache2 apache2-dev libev-dev gcc python-setuptools
+
+# NginX Unit part, start:
+RUN apt-get -y  --no-install-recommends install software-properties-common apt-transport-https gnupg2 dirmngr curl
+RUN add-apt-repository 'deb https://packages.nginx.org/unit/debian/ stretch unit'
+RUN apt-key adv --fetch-keys https://nginx.org/keys/nginx_signing.key
+RUN apt-get -y update
+RUN apt-get -y  --no-install-recommends install unit-python2.7
+# NginX Unit part, end.
+
+RUN rm -rf /var/lib/apt/lists/*
+RUN pip install \
+                cherrypy==17.4.0    \
+                uwsgi==2.0.17.1     \
+                gunicorn==19.9.0    \
+                bjoern==2.2.3       \
+                meinheld==0.6.1     \
+                mod_wsgi==4.6.5
 RUN mkdir -p /home/www
 COPY src /home/www/wsgi_benchmark
 RUN chown -R www-data:www-data /home/www/wsgi_benchmark

benchmark.sh

@@ -13,10 +13,10 @@ function start() {
     local id=$1
     shift
     echo "Starting $id ($*)"
-    CONTAINER=$(docker run --detach --memory 512MB --cpuset-cpus 0,1 --workdir /home/www/wsgi_benchmark -p $PORT:$PORT wsgi_benchmark "$@")
+    CONTAINER=$(docker run --detach --rm --memory 512MB --cpuset-cpus 0,1 --workdir /home/www/wsgi_benchmark -p $PORT:$PORT wsgi_benchmark "$@")
 
     while true; do
-        echo "    Waiting for container ..."
+        echo "    Waiting for container ... $CONTAINER"
         IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' "$CONTAINER")
         result=$(curl --silent "http://$IP:$PORT")
         if [[ "$result" == "OK" ]]; then
@@ -32,14 +32,15 @@ function stop() {
 }
 
 function perf() {
-    docker stats > "$BASE/$1.$2.stats" &
+    docker stats --no-trunc --format "{{.CPUPerc}} {{.MemUsage}}" > "$BASE/$1.$2.stats" &
     STATS_PID=$!
 
     echo "    Testing $1 with $2 connections ..."
     taskset -c 2,3 wrk --duration 30s --threads 4 --connections "$2" "http://$IP:$PORT" > "$BASE/$1.$2.log"
 
     sleep 1
     kill $STATS_PID
+    wait $STATS_PID 2>/dev/null
 }
 
 # Install docker image (if needed)

readme.md

@@ -14,9 +14,15 @@ Pull requests are welcome!
 Steps to reproduce benchmarks
 -----------------------------
 
-### 1. Install docker ###
+### 1a. Install docker ###
 
 Please consult instructions for your distro
+The test was performed with docker version 18.06.1-ce.
+
+### 1b. Install wrk
+
+Please consult instructions for your distro
+The test was performed with wrk version 4.1.0
 
 ### 2. Run benchmark.sh ###
 

results.csv

@@ -1,45 +1,55 @@
 
 REQUESTS
 server,100,200,300,400,500,600,700,800,900,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000
-Bjoern,61476.445,128594.775,56013.075,111486.39,105241.825,102233.64,99782.15,100013.52,96742.22,96561.15,89824.7,80815.04,65394.995,58352.365,68084.015,76935.605,81163.475,67023.45,82331.875
-CherryPy,4090.07,4077.545,4069.04,4047.695,4084.915,4081.215,4072.97,4080.11,4056.755,4088.54,4079.985,4072.005,4049.775,3817.44,3759.135,4023.95,4040.17,4033.175,4004.185
-Gunicorn,1044.755,616.2,1370.205,655.335,279.78,1540.485,1248.215,1135.545,1070.82,1403.395,1540.3,1114.545,1455.79,1076.35,1014.055,980.405,1032.63,723.245,806.76
-Meinheld,1129.065,1419.15,2220.555,1079.455,2667.67,155.415,1005.9,2106.45,2734.265,2520.695,2085.62,2317.425,3844.27,2975.69,3768.505,3149.115,2413.38,2201.0,2732.755
-mod_WSGI-express,1254.34,0.33,7.795,1887.885,1943.23,1986.5,1558.695,1957.38,1884.415,1766.99,1669.55,1686.485,1568.665,1753.56,1862.34,1771.21,1806.31,1661.4,1747.9
-uWSGI,3.475,7.265,11.77,16.325,20.075,21.63,27.02,28.385,32.155,36.495,69.75,106.1,138.485,170.36,204.115,239.905,273.96,306.36,336.225
+Bjoern,57124.82,56450.065,53915.845,52002.65,50236.68,49082.205,48964.72,48368.59,47069.55,48080.64,34451.17,36352.27,42734.07,44474.345,46587.51,45917.025,46115.28,46447.52,46318.415
+CherryPy,1933.37,1922.505,1934.04,1939.54,1929.825,1930.29,1935.115,1921.04,1920.395,1921.35,1868.86,1857.53,1885.555,1882.835,1855.025,1859.65,1845.275,1873.305,1657.795
+Gunicorn,1034.63,1055.82,870.16,755.71,951.965,766.385,918.135,748.34,980.12,995.585,760.06,839.4,1009.27,628.0,831.915,784.885,885.82,846.525,537.83
+Meinheld,1013.94,1319.585,1556.69,1669.725,2031.945,1844.095,2335.83,2026.545,2059.865,1972.98,2016.235,2131.645,2286.695,2317.96,1933.755,2441.235,2598.145,1909.03,2124.765
+mod_WSGI-express,980.535,1189.13,649.09,1091.995,727.585,1054.29,910.47,1130.015,985.745,1093.08,1002.37,1144.475,819.755,1118.68,1156.94,1169.835,1174.1,781.14,1082.505
+nginx-unit,17309.555,16586.71,16607.215,16480.985,16513.045,16314.025,14717.585,14748.435,14736.515,14632.205,13929.47,15047.525,15332.09,14913.24,14949.965,14296.1,15362.57,14200.695,14652.295
+uWSGI,29121.39,29074.805,29083.615,28779.635,29125.205,29074.78,29006.15,29260.28,28608.375,28799.84,28911.42,28750.525,29058.305,29034.415,28768.61,28845.575,28852.645,28925.16,28446.075
+uWSGI-base,29380.12,29089.99,29309.96,29078.395,29473.42,29158.645,29170.24,28939.5,29123.18,29252.035,29215.15,29209.845,29169.72,29094.8,29090.285,29024.66,29036.265,28934.31,28845.29
 
 LATENCIES
 server,100,200,300,400,500,600,700,800,900,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000
-Bjoern,0.395445,1.565,0.06,3.57,4.775,5.88,7.03,6.86,8.63,9.33,18.835,25.165,44.155,58.6,69.38,81.475,91.74,84.135,116.265
-CherryPy,2.44,2.435,2.43,2.43,2.445,2.445,2.445,2.44,2.455,2.435,2.445,2.445,2.45,2.61,2.435,2.465,2.455,2.445,2.47
-Gunicorn,26.245,40.715,48.23,51.26,71.05,58.645,50.67,52.39,82.18,74.38,63.005,100.78,72.425,71.115,95.625,84.5,106.155,81.385,143.24
-Meinheld,15.265,32.765,60.51,61.045,90.125,57.935,97.085,146.895,146.3,169.41,316.275,461.595,389.285,520.24,395.64,482.26,562.465,626.745,482.34
-mod_WSGI-express,24.275,3.6615,18.075,96.445,115.07,130.92,130.965,158.15,156.82,133.45,165.22,201.075,252.655,231.36,197.3,201.26,210.23,246.865,231.99
-uWSGI,8.065,6.73,73.045,42.12,47.395,37.29,62.77,197.875,59.765,295.93,423.995,726.94,667.505,611.83,596.94,741.71,728.7,747.145,881.165
+Bjoern,1.735,3.535,5.5,7.75,9.71,12.5,14.54,16.88,19.945,20.47,57.07,79.015,89.21,96.79,121.43,141.765,146.9,171.665,189.07
+CherryPy,5.145,5.19,5.16,5.14,5.165,5.16,5.135,5.16,5.19,5.185,5.335,5.29,5.265,5.225,5.3,5.29,5.32,5.28,5.32
+Gunicorn,58.59,80.19,83.695,64.51,73.32,103.335,86.18,69.665,100.38,91.99,86.395,113.63,136.875,181.425,151.975,205.435,193.395,197.125,248.06
+Meinheld,50.79,76.115,95.475,112.405,109.605,145.62,127.915,177.45,180.335,203.945,313.805,485.85,509.295,580.755,732.715,584.705,524.015,780.72,791.85
+mod_WSGI-express,59.755,93.755,245.48,160.23,258.515,177.835,189.02,154.215,195.395,164.045,213.855,164.82,277.79,203.405,206.51,219.84,239.225,282.39,246.64
+nginx-unit,5.755,12.0,18.055,24.09,29.845,36.485,46.76,53.62,60.085,67.38,132.455,193.005,254.385,313.325,385.31,445.54,458.0,601.99,597.915
+uWSGI,0.33293,0.33429,0.335415,0.33692,0.334935,0.33563,0.334425,0.333285,0.3344,0.333585,0.334505,0.335325,0.33479,0.333825,0.33772,0.33276,0.333665,0.3357,0.336485
+uWSGI-base,0.16046,0.163775,0.16417,0.16348,0.162775,0.16436,0.16453,0.16299,0.16461,0.163825,0.16357,0.16365,0.163065,0.164345,0.16311,0.164315,0.16351,0.16438,0.16482
 
 CPU
 server,100,200,300,400,500,600,700,800,900,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000
-Bjoern,51.13,102.34,51.37,102.115,101.73,102.78,101.34,101.35,101.69,101.725,101.86,101.995,101.335,102.365,106.86,106.77,108.665,108.06,107.005
-CherryPy,143.755,142.68,141.9,142.695,142.525,142.625,142.78,143.715,142.64,142.615,142.515,142.845,142.66,141.955,142.545,142.91,142.31,142.36,143.125
-Gunicorn,169.415,98.69,179.56,116.59,44.75,192.085,195.8,180.39,164.55,165.39,182.775,165.805,178.97,157.58,164.5,162.29,170.585,181.42,117.54
-Meinheld,93.26,102.53,102.15,56.535,102.31,13.92,38.105,85.035,99.56,98.435,102.24,101.6,97.88,104.57,103.6,96.885,98.19,97.005,100.475
-mod_WSGI-express,199.89,0.545,11.13,201.18,199.775,199.33,197.825,200.995,199.6,199.455,200.45,199.5,197.645,200.58,201.815,202.44,202.95,204.0,199.48
-uWSGI,0,0,1.49,0.81,1.485,1.98,2.295,3.54,4.59,6.045,13.49,16.205,23.935,28.08,34.05,43.095,37.01,40.19,45.635
+Bjoern,62.03,61.66,62.265,62.925,63.095,63.195,63.955,64.235,64.425,64.79,65.875,61.47,66.625,68.595,70.5,70.485,71.02,70.84,70.87
+CherryPy,133.255,133.635,133.47,133.385,133.47,133.34,133.555,133.66,133.27,133.49,133.02,132.975,133.84,133.46,132.99,133.02,133.455,133.385,133.17
+Gunicorn,172.49,172.78,173.075,171.775,172.645,164.95,172.675,172.45,171.715,173.215,173.56,171.38,173.34,166.605,173.29,152.775,166.91,157.325,132.83
+Meinheld,78.33,80.05,79.675,79.205,79.91,79.07,79.25,79.105,79.52,79.325,79.715,79.88,78.955,80.035,77.395,77.72,79.795,73.37,75.805
+mod_WSGI-express,179.44,180.12,150.13,179.97,165.885,180.525,180.74,180.72,180.0,180.43,178.46,180.8,172.9,181.405,180.405,180.86,176.77,137.595,164.18
+nginx-unit,161.555,160.715,159.98,159.555,158.895,160.91,153.0,153.855,152.81,153.505,150.07,158.015,162.335,162.0,161.975,161.865,160.23,163.315,160.465
+uWSGI,149.675,149.65,149.77,150.535,150.475,150.35,150.16,149.975,149.83,149.255,150.145,150.16,150.05,150.105,150.125,150.4,149.84,149.945,150.13
+uWSGI-base,148.13,148.455,148.545,149.575,150.465,149.15,148.575,149.04,149.06,149.725,149.605,149.405,149.435,148.895,148.735,149.285,149.62,148.935,148.725
 
 MEMORY
 server,100,200,300,400,500,600,700,800,900,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000
-Bjoern,3.5575,3.572,3.449,3.684,3.7175,3.8505,3.869,3.8315,3.949,3.932,4.479,4.69,5.5195,5.7425,6.4165,7.0635,7.69,8.0365,9.0545
-CherryPy,12.96,14.17,13.52,13.85,14.27,14.545,14.73,15.295,16.07,16.03,19.57,24.15,25.625,28.125,30.54,36.335,41.395,42.565,45.695
-Gunicorn,21.59,21.565,21.595,21.615,21.625,22.35,22.09,22.045,21.605,22.8,23.015,23.865,22.65,24.01,22.455,23.71,23.635,22.14,23.765
-Meinheld,6.0025,6.0845,6.216,6.0645,6.439,6.058,5.9945,6.111,6.3735,6.22,7.0165,6.752,6.785,6.93,7.422,7.555,7.92,8.053,9.1465
-mod_WSGI-express,21.58,15.725,16.685,29.07,30.66,31.32,29.675,37.775,34.94,32.245,37.69,37.16,45.595,33.5,34.595,32.785,35.925,40.425,43.285
-uWSGI,16.29,15.9,15.925,15.9,15.875,15.935,15.875,15.915,15.97,15.925,27.805,50.17,58.31,67.04,68.985,87.925,105.9,114.2,118.9
+Bjoern,4.974444,5.0913605,5.38234,5.514985,5.679087,5.900337,6.070206,6.285164,6.4676165,6.6516415,8.7246765,10.915676,13.0233135,14.25539,16.719544,18.9634965,19.2570975,22.1931105,23.876075
+CherryPy,15.151923,16.3892425,17.5112185,18.53358,19.6503135,20.761804,21.9571805,23.100129,24.0648185,25.108152,36.333158,44.768952,48.4913965,52.376371,55.6636565,60.82265,79.9853765,72.43563,102.7761765
+Gunicorn,30.7914335,30.83862,30.8858055,30.8858055,30.8910485,30.8438625,30.864834,30.859591,30.817648,30.8386195,30.8386195,30.8491055,30.922506,30.7599765,30.8858055,30.8543485,30.870077,30.8543485,30.833377
+Meinheld,7.133462,7.446462,7.8952525,7.9565945,8.052539,8.107589,8.196194,8.1427165,8.5584765,8.8164265,9.2982475,10.873733,12.158238,13.967032,15.089008,15.2725085,17.065574,18.465423,20.4891745
+mod_WSGI-express,27.0008315,30.7652195,32.4691555,33.0353865,33.664532,32.3013835,31.8033095,32.123125,32.956743,33.3552025,31.2318355,31.598837,32.3747835,31.577866,31.090278,31.745638,31.614566,33.659289,33.003929
+nginx-unit,54.4368225,53.7500055,54.137978,55.474913,56.806604,56.6965035,54.767124,55.585013,56.334745,57.8499375,65.4992995,76.451676,86.9951075,95.9866465,105.0830435,114.976358,125.671833,131.3341435,143.183052
+uWSGI,18.779996,18.795724,18.596495,18.685624,18.722324,18.6698955,18.643681,18.6908665,18.7432955,18.7013525,18.7957245,18.8219385,18.753781,18.816696,18.800967,18.7799955,18.816696,18.759024,18.7432955
+uWSGI-base,18.2137645,18.229493,18.203279,18.229493,18.1246355,18.2557075,18.203279,18.3028935,18.2137645,18.1718215,18.2661935,18.3395935,18.334351,18.3867795,18.334351,18.407751,18.334351,18.381537,18.3867795
 
 ERRORS
 server,100,200,300,400,500,600,700,800,900,1000,2000,3000,4000,5000,6000,7000,8000,9000,10000
-Bjoern,50,0,293,0,0,0,0,99,46,109,382,610,752,1162,658,543,241,1600,227
-CherryPy,0,0,0,0,0,0,55,0,0,0,0,0,0,786,865,145,0,0,0
-Gunicorn,100,212,467,472,500,679,652,741,693,814,863,1287,1011,858,1509,1280,1274,446,2106
-Meinheld,100,240,177,295,178,600,518,320,388,386,1189,2163,4198,5578,8301,7989,8970,12046,10593
-mod_WSGI-express,100,200,300,244,275,334,448,556,630,713,1375,1141,2455,1866,2350,2372,2659,3039,3255
-uWSGI,4,29,54,91,104,79,113,54,67,119,151,819,1265,1508,1311,2298,3382,3970,3437
+Bjoern,0,0,0,0,0,0,0,0,0,0,328,37,9,455,372,263,394,346,307
+CherryPy,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+Gunicorn,0,101,243,363,346,284,379,468,335,485,677,688,714,768,1134,943,1214,1316,1033
+Meinheld,0,53,120,137,191,305,349,157,544,596,1015,1279,1795,3375,7428,2549,3636,9175,2314
+mod_WSGI-express,0,34,2,200,66,270,261,337,320,364,729,763,1091,1102,1148,1344,1635,1501,1762
+nginx-unit,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+uWSGI,0,13,25,51,41,50,50,50,50,50,50,50,51,50,50,51,51,51,50
+uWSGI-base,0,14,17,50,50,50,50,50,51,50,51,50,50,50,50,50,50,50,50

results.py

@@ -4,11 +4,11 @@
 import re
 import os
 import sys
-import glob
-import pprint
+import humanfriendly
 
 from collections import defaultdict
 
+
 class Results(object):
     def __init__(self):
         self.LATENCIES = defaultdict(dict)
@@ -17,8 +17,8 @@ def __init__(self):
         self.MEMORY = defaultdict(lambda: defaultdict(int))
         self.CPU = defaultdict(lambda: defaultdict(int))
 
-class Parser(object):
 
+class Parser(object):
     RESULTS = defaultdict(Results)
     DIGITS_RE = re.compile(r'\d+')
     LATENCY_RE = re.compile(r'([\d\.]+)(\w+)')
@@ -49,27 +49,25 @@ def _fileHandler(self, fileName):
 
     def _logHandler(self, directory, server, connections, content):
         for line in content:
-            parts = [ part for part in line.split(' ') if part ]
+            parts = [part for part in line.split(' ') if part]
             if parts[0] == 'Latency':
                 digits, unit = self.LATENCY_RE.match(parts[1]).groups()
                 self.RESULTS[directory].LATENCIES[server][connections] = float(digits) * self.LATENCY_MULTIPLIERS[unit]
             elif 'Requests' in parts[0]:
                 self.RESULTS[directory].REQUESTS[server][connections] = float(parts[1])
             elif 'Socket' == parts[0]:
-                self.RESULTS[directory].ERRORS[server][connections] = sum(
-                    int(i) for i in self.DIGITS_RE.findall(line)
-                )
+                self.RESULTS[directory].ERRORS[server][connections] = sum([int(i) for i in self.DIGITS_RE.findall(line)])
 
     def _statsHandler(self, directory, server, connections, content):
         for line in content:
-            parts = [ part for part in line.split(' ') if part ]
-            if 'CONTAINER' not in parts[0]:
-                cpu = float(parts[1].rstrip('%'))
-                memory = float(parts[2])
-                if cpu > self.RESULTS[directory].CPU[server][connections]:
-                    self.RESULTS[directory].CPU[server][connections] = cpu
-                if memory > self.RESULTS[directory].MEMORY[server][connections]:
-                    self.RESULTS[directory].MEMORY[server][connections] = memory
+            parts = [part for part in line.split(' ') if part]
+            cpu = float(parts[0].rstrip('%').lstrip('\x1b[2J\x1b[H'))
+            memory = humanfriendly.parse_size(parts[1])/1000000.0
+            if cpu > self.RESULTS[directory].CPU[server][connections]:
+                self.RESULTS[directory].CPU[server][connections] = cpu
+            if memory > self.RESULTS[directory].MEMORY[server][connections]:
+                self.RESULTS[directory].MEMORY[server][connections] = memory
+
 
 class Output(object):
 
@@ -100,7 +98,7 @@ def _best(self, data, metric, server, count):
         )[1:-1]
 
         # Average the remaining values
-        return sum(values)/len(values)
+        return sum(values) / len(values)
 
     def _requests(self, results):
         print()
@@ -172,6 +170,7 @@ def _errors(self, results):
                 )
             )
 
+
 if __name__ == '__main__':
     parser = Parser()
     for directory in sys.argv[1:]:

round1/Bjoern.100.log

@@ -1,9 +1,8 @@
 Running 30s test @ http://172.17.0.2:9808
   4 threads and 100 connections
   Thread Stats   Avg      Stdev     Max   +/- Stdev
-    Latency     0.00us    0.00us   0.00us    -nan%
-    Req/Sec     0.00      0.00     0.00      -nan%
-  0 requests in 30.09s, 0.00B read
-  Socket errors: connect 100, read 0, write 0, timeout 0
-Requests/sec:      0.00
-Transfer/sec:       0.00B
+    Latency     1.77ms  390.82us  50.96ms   99.03%
+    Req/Sec    14.12k   398.45    14.74k    95.75%
+  1686112 requests in 30.01s, 144.72MB read
+Requests/sec:  56182.17
+Transfer/sec:      4.82MB