Better performance measurements using AWS cloud

This commit is contained in:
Dennis Schwerdel 2020-06-08 16:16:11 +02:00
parent 689d00ba20
commit 717c360ef1
7 changed files with 1522 additions and 21 deletions

21
perf.sh
View File

@ -1,21 +0,0 @@
#!/bin/bash
DST=$1
echo
echo "----------"
echo "Throughput"
echo "----------"
for i in 0 1 2; do
iperf -c $DST -t 30
done
for size in 100 500 1000; do
echo
echo "--------------------"
echo "Latency ($size Bytes)"
echo "--------------------"
for i in 0 1 2 3 4; do
ping $DST -c 30000 -i 0.001 -s $size -U -q
done
done

View File

@ -0,0 +1,227 @@
{
"aes256-7000": {
"ping_500": {
"rtt_min": 0.074,
"rtt_avg": 0.088,
"pkt_loss": 0.0,
"rtt_max": 13.643
},
"iperf": {
"cpu_sender": 6.394363,
"cpu_receiver": 54.566188,
"throughput": 2736147000.0
},
"ping_100": {
"rtt_min": 0.073,
"rtt_avg": 0.085,
"pkt_loss": 0.0,
"rtt_max": 0.698
},
"ping_1000": {
"rtt_min": 0.077,
"rtt_avg": 0.089,
"pkt_loss": 0.0,
"rtt_max": 0.152
}
},
"results": {
"throughput_mbits": {
"aes256-7000": 2736.147,
"plain-1400": 914.593510048459,
"chacha20-1400": 660.0519042921719,
"chacha20-7000": 2250.026,
"aes256-1400": 761.187334946869,
"plain-7000": 3930.962,
"native": 9678.675
},
"latency_ms": {
"aes256-7000": {
"100": 20.000000000000004,
"500": 20.499999999999996,
"1000": 21.5
},
"plain-1400": {
"100": 19.000000000000004,
"500": 19.000000000000004,
"1000": 20.499999999999996
},
"chacha20-1400": {
"100": 22.0,
"500": 23.0,
"1000": 26.500000000000004
},
"chacha20-7000": {
"100": 21.5,
"500": 23.5,
"1000": 26.500000000000004
},
"aes256-1400": {
"100": 19.000000000000004,
"500": 19.000000000000004,
"1000": 21.5
},
"plain-7000": {
"100": 18.500000000000004,
"500": 19.000000000000004,
"1000": 19.999999999999996
}
}
},
"plain-1400": {
"ping_500": {
"rtt_min": 0.073,
"rtt_avg": 0.085,
"pkt_loss": 0.0,
"rtt_max": 13.084
},
"iperf": {
"cpu_sender": 2.973476,
"cpu_receiver": 70.948196,
"throughput": 914593510.048459
},
"ping_100": {
"rtt_min": 0.066,
"rtt_avg": 0.083,
"pkt_loss": 0.0,
"rtt_max": 13.291
},
"ping_1000": {
"rtt_min": 0.073,
"rtt_avg": 0.087,
"pkt_loss": 0.0,
"rtt_max": 13.257
}
},
"chacha20-1400": {
"ping_500": {
"rtt_min": 0.08,
"rtt_avg": 0.093,
"pkt_loss": 0.0,
"rtt_max": 13.11
},
"iperf": {
"cpu_sender": 2.598758,
"cpu_receiver": 65.3863,
"throughput": 660051904.292172
},
"ping_100": {
"rtt_min": 0.071,
"rtt_avg": 0.089,
"pkt_loss": 0.0,
"rtt_max": 13.328
},
"ping_1000": {
"rtt_min": 0.084,
"rtt_avg": 0.099,
"pkt_loss": 0.0,
"rtt_max": 13.41
}
},
"meta": {
"ami": "ami-0a02ee601d742e89f",
"instance_type": "m5.large",
"region": "eu-central-1",
"version": "1.0.0",
"duration": 870.1210024356842
},
"chacha20-7000": {
"ping_500": {
"rtt_min": 0.079,
"rtt_avg": 0.094,
"pkt_loss": 0.0,
"rtt_max": 13.602
},
"iperf": {
"cpu_sender": 5.441034,
"cpu_receiver": 58.117002,
"throughput": 2250026000.0
},
"ping_100": {
"rtt_min": 0.07,
"rtt_avg": 0.088,
"pkt_loss": 0.0,
"rtt_max": 0.186
},
"ping_1000": {
"rtt_min": 0.085,
"rtt_avg": 0.099,
"pkt_loss": 0.0,
"rtt_max": 1.131
}
},
"aes256-1400": {
"ping_500": {
"rtt_min": 0.073,
"rtt_avg": 0.085,
"pkt_loss": 0.0,
"rtt_max": 12.865
},
"iperf": {
"cpu_sender": 2.531172,
"cpu_receiver": 64.35578,
"throughput": 761187334.946869
},
"ping_100": {
"rtt_min": 0.069,
"rtt_avg": 0.083,
"pkt_loss": 0.0,
"rtt_max": 13.247
},
"ping_1000": {
"rtt_min": 0.075,
"rtt_avg": 0.089,
"pkt_loss": 0.0,
"rtt_max": 13.338
}
},
"plain-7000": {
"ping_500": {
"rtt_min": 0.073,
"rtt_avg": 0.085,
"pkt_loss": 0.0,
"rtt_max": 13.48
},
"iperf": {
"cpu_sender": 9.545279,
"cpu_receiver": 66.830757,
"throughput": 3930962000.0
},
"ping_100": {
"rtt_min": 0.07,
"rtt_avg": 0.082,
"pkt_loss": 0.0,
"rtt_max": 0.2
},
"ping_1000": {
"rtt_min": 0.075,
"rtt_avg": 0.086,
"pkt_loss": 0.0,
"rtt_max": 0.927
}
},
"native": {
"ping_500": {
"rtt_min": 0.04,
"rtt_avg": 0.047,
"pkt_loss": 0.0,
"rtt_max": 12.794
},
"iperf": {
"cpu_sender": 14.622503,
"cpu_receiver": 74.530595,
"throughput": 9678675000.0
},
"ping_100": {
"rtt_min": 0.038,
"rtt_avg": 0.045,
"pkt_loss": 0.0,
"rtt_max": 13.168
},
"ping_1000": {
"rtt_min": 0.041,
"rtt_avg": 0.046,
"pkt_loss": 0.0,
"rtt_max": 13.177
}
}
}

View File

@ -0,0 +1,227 @@
{
"aes256-7000": {
"ping_500": {
"rtt_min": 0.074,
"rtt_avg": 0.086,
"pkt_loss": 0.0,
"rtt_max": 0.585
},
"iperf": {
"cpu_sender": 5.708496,
"cpu_receiver": 58.180282,
"throughput": 3004232000.0
},
"ping_100": {
"rtt_min": 0.071,
"rtt_avg": 0.084,
"pkt_loss": 0.0,
"rtt_max": 0.691
},
"ping_1000": {
"rtt_min": 0.073,
"rtt_avg": 0.087,
"pkt_loss": 0.0,
"rtt_max": 0.633
}
},
"results": {
"throughput_mbits": {
"aes256-7000": 3004.232,
"plain-1400": 937.318890315313,
"chacha20-1400": 616.982586318721,
"chacha20-7000": 2139.102,
"aes256-1400": 804.276052366516,
"plain-7000": 4251.583,
"native": 9677.49
},
"latency_ms": {
"aes256-7000": {
"100": 19.500000000000004,
"500": 19.999999999999996,
"1000": 19.999999999999996
},
"plain-1400": {
"100": 18.500000000000004,
"500": 19.000000000000004,
"1000": 19.000000000000004
},
"chacha20-1400": {
"100": 21.5,
"500": 24.0,
"1000": 26.000000000000004
},
"chacha20-7000": {
"100": 21.5,
"500": 23.5,
"1000": 25.0
},
"aes256-1400": {
"100": 19.500000000000004,
"500": 20.499999999999996,
"1000": 19.999999999999996
},
"plain-7000": {
"100": 18.500000000000004,
"500": 18.500000000000004,
"1000": 19.000000000000004
}
}
},
"plain-1400": {
"ping_500": {
"rtt_min": 0.069,
"rtt_avg": 0.084,
"pkt_loss": 0.0,
"rtt_max": 0.559
},
"iperf": {
"cpu_sender": 2.846318,
"cpu_receiver": 65.526508,
"throughput": 937318890.315313
},
"ping_100": {
"rtt_min": 0.064,
"rtt_avg": 0.082,
"pkt_loss": 0.0,
"rtt_max": 3.378
},
"ping_1000": {
"rtt_min": 0.072,
"rtt_avg": 0.085,
"pkt_loss": 0.0,
"rtt_max": 0.529
}
},
"chacha20-1400": {
"ping_500": {
"rtt_min": 0.081,
"rtt_avg": 0.094,
"pkt_loss": 0.0,
"rtt_max": 0.537
},
"iperf": {
"cpu_sender": 2.220085,
"cpu_receiver": 59.560219,
"throughput": 616982586.318721
},
"ping_100": {
"rtt_min": 0.073,
"rtt_avg": 0.088,
"pkt_loss": 0.0,
"rtt_max": 0.641
},
"ping_1000": {
"rtt_min": 0.085,
"rtt_avg": 0.099,
"pkt_loss": 0.0,
"rtt_max": 0.67
}
},
"meta": {
"ami": "ami-0a02ee601d742e89f",
"instance_type": "m5.large",
"region": "eu-central-1",
"version": "1.1.0",
"duration": 870.0856907367706
},
"chacha20-7000": {
"ping_500": {
"rtt_min": 0.079,
"rtt_avg": 0.093,
"pkt_loss": 0.0,
"rtt_max": 0.397
},
"iperf": {
"cpu_sender": 4.693588,
"cpu_receiver": 51.336932,
"throughput": 2139102000.0
},
"ping_100": {
"rtt_min": 0.074,
"rtt_avg": 0.088,
"pkt_loss": 0.0,
"rtt_max": 0.569
},
"ping_1000": {
"rtt_min": 0.084,
"rtt_avg": 0.097,
"pkt_loss": 0.0,
"rtt_max": 0.572
}
},
"aes256-1400": {
"ping_500": {
"rtt_min": 0.071,
"rtt_avg": 0.087,
"pkt_loss": 0.0,
"rtt_max": 10.58
},
"iperf": {
"cpu_sender": 2.510188,
"cpu_receiver": 64.157275,
"throughput": 804276052.366516
},
"ping_100": {
"rtt_min": 0.067,
"rtt_avg": 0.084,
"pkt_loss": 0.0,
"rtt_max": 1.54
},
"ping_1000": {
"rtt_min": 0.071,
"rtt_avg": 0.087,
"pkt_loss": 0.0,
"rtt_max": 0.237
}
},
"plain-7000": {
"ping_500": {
"rtt_min": 0.068,
"rtt_avg": 0.083,
"pkt_loss": 0.0,
"rtt_max": 0.402
},
"iperf": {
"cpu_sender": 8.801774,
"cpu_receiver": 66.18731,
"throughput": 4251583000.0
},
"ping_100": {
"rtt_min": 0.067,
"rtt_avg": 0.082,
"pkt_loss": 0.0,
"rtt_max": 0.701
},
"ping_1000": {
"rtt_min": 0.07,
"rtt_avg": 0.085,
"pkt_loss": 0.0,
"rtt_max": 0.266
}
},
"native": {
"ping_500": {
"rtt_min": 0.041,
"rtt_avg": 0.046,
"pkt_loss": 0.0,
"rtt_max": 0.52
},
"iperf": {
"cpu_sender": 14.901057,
"cpu_receiver": 61.512449,
"throughput": 9677490000.0
},
"ping_100": {
"rtt_min": 0.038,
"rtt_avg": 0.045,
"pkt_loss": 0.0,
"rtt_max": 0.651
},
"ping_1000": {
"rtt_min": 0.041,
"rtt_avg": 0.047,
"pkt_loss": 0.0,
"rtt_max": 0.663
}
}
}

View File

@ -0,0 +1,227 @@
{
"aes256-7000": {
"ping_500": {
"rtt_min": 0.077,
"rtt_avg": 0.091,
"pkt_loss": 0.0,
"rtt_max": 0.22
},
"iperf": {
"cpu_sender": 4.870841,
"cpu_receiver": 58.056654,
"throughput": 2906372000.0
},
"ping_100": {
"rtt_min": 0.076,
"rtt_avg": 0.09,
"pkt_loss": 0.0,
"rtt_max": 0.597
},
"ping_1000": {
"rtt_min": 0.078,
"rtt_avg": 0.093,
"pkt_loss": 0.0,
"rtt_max": 0.925
}
},
"results": {
"throughput_mbits": {
"aes256-7000": 2906.372,
"plain-1400": 845.4989025966671,
"chacha20-1400": 593.980680023114,
"chacha20-7000": 2290.887,
"aes256-1400": 752.968939682118,
"plain-7000": 4438.046,
"native": 9680.156
},
"latency_ms": {
"aes256-7000": {
"100": 22.0,
"500": 21.5,
"1000": 22.0
},
"plain-1400": {
"100": 20.499999999999996,
"500": 20.499999999999996,
"1000": 20.999999999999996
},
"chacha20-1400": {
"100": 23.5,
"500": 25.500000000000004,
"1000": 27.499999999999996
},
"chacha20-7000": {
"100": 23.5,
"500": 25.500000000000004,
"1000": 26.999999999999996
},
"aes256-1400": {
"100": 21.5,
"500": 22.0,
"1000": 22.5
},
"plain-7000": {
"100": 20.499999999999996,
"500": 20.499999999999996,
"1000": 20.499999999999996
}
}
},
"plain-1400": {
"ping_500": {
"rtt_min": 0.074,
"rtt_avg": 0.089,
"pkt_loss": 0.0,
"rtt_max": 3.605
},
"iperf": {
"cpu_sender": 1.925076,
"cpu_receiver": 65.76581,
"throughput": 845498902.596667
},
"ping_100": {
"rtt_min": 0.071,
"rtt_avg": 0.087,
"pkt_loss": 0.0,
"rtt_max": 0.381
},
"ping_1000": {
"rtt_min": 0.077,
"rtt_avg": 0.091,
"pkt_loss": 0.0,
"rtt_max": 3.54
}
},
"chacha20-1400": {
"ping_500": {
"rtt_min": 0.085,
"rtt_avg": 0.099,
"pkt_loss": 0.0,
"rtt_max": 1.654
},
"iperf": {
"cpu_sender": 1.538928,
"cpu_receiver": 59.900258,
"throughput": 593980680.023114
},
"ping_100": {
"rtt_min": 0.074,
"rtt_avg": 0.093,
"pkt_loss": 0.0,
"rtt_max": 0.382
},
"ping_1000": {
"rtt_min": 0.088,
"rtt_avg": 0.104,
"pkt_loss": 0.0,
"rtt_max": 3.686
}
},
"meta": {
"ami": "ami-0a02ee601d742e89f",
"instance_type": "m5.large",
"region": "eu-central-1",
"version": "1.2.0",
"duration": 870.1119563579559
},
"chacha20-7000": {
"ping_500": {
"rtt_min": 0.082,
"rtt_avg": 0.099,
"pkt_loss": 0.0,
"rtt_max": 0.702
},
"iperf": {
"cpu_sender": 4.365811,
"cpu_receiver": 57.666402,
"throughput": 2290887000.0
},
"ping_100": {
"rtt_min": 0.075,
"rtt_avg": 0.093,
"pkt_loss": 0.0,
"rtt_max": 0.591
},
"ping_1000": {
"rtt_min": 0.09,
"rtt_avg": 0.103,
"pkt_loss": 0.0,
"rtt_max": 0.352
}
},
"aes256-1400": {
"ping_500": {
"rtt_min": 0.077,
"rtt_avg": 0.092,
"pkt_loss": 0.0,
"rtt_max": 0.239
},
"iperf": {
"cpu_sender": 1.643239,
"cpu_receiver": 64.53098,
"throughput": 752968939.682118
},
"ping_100": {
"rtt_min": 0.075,
"rtt_avg": 0.089,
"pkt_loss": 0.0,
"rtt_max": 1.615
},
"ping_1000": {
"rtt_min": 0.08,
"rtt_avg": 0.094,
"pkt_loss": 0.0,
"rtt_max": 0.676
}
},
"plain-7000": {
"ping_500": {
"rtt_min": 0.074,
"rtt_avg": 0.089,
"pkt_loss": 0.0,
"rtt_max": 0.586
},
"iperf": {
"cpu_sender": 13.25846,
"cpu_receiver": 68.481618,
"throughput": 4438046000.0
},
"ping_100": {
"rtt_min": 0.071,
"rtt_avg": 0.087,
"pkt_loss": 0.0,
"rtt_max": 0.503
},
"ping_1000": {
"rtt_min": 0.076,
"rtt_avg": 0.09,
"pkt_loss": 0.0,
"rtt_max": 0.234
}
},
"native": {
"ping_500": {
"rtt_min": 0.042,
"rtt_avg": 0.048,
"pkt_loss": 0.0,
"rtt_max": 0.305
},
"iperf": {
"cpu_sender": 19.960378,
"cpu_receiver": 65.885621,
"throughput": 9680156000.0
},
"ping_100": {
"rtt_min": 0.041,
"rtt_avg": 0.046,
"pkt_loss": 0.0,
"rtt_max": 0.312
},
"ping_1000": {
"rtt_min": 0.043,
"rtt_avg": 0.049,
"pkt_loss": 0.0,
"rtt_max": 0.509
}
}
}

View File

@ -0,0 +1,227 @@
{
"meta": {
"region": "eu-central-1",
"instance_type": "m5.large",
"ami": "ami-0a02ee601d742e89f",
"version": "1.3.0",
"duration": 869.9226264953613
},
"native": {
"iperf": {
"throughput": 9678121000.0,
"cpu_sender": 18.541256,
"cpu_receiver": 65.911053
},
"ping_100": {
"rtt_min": 0.047,
"rtt_max": 0.113,
"rtt_avg": 0.054,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.05,
"rtt_max": 0.14,
"rtt_avg": 0.055,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.049,
"rtt_max": 0.173,
"rtt_avg": 0.056,
"pkt_loss": 0.0
}
},
"plain-1400": {
"iperf": {
"throughput": 937787147.728354,
"cpu_sender": 3.517415,
"cpu_receiver": 66.394897
},
"ping_100": {
"rtt_min": 0.074,
"rtt_max": 0.242,
"rtt_avg": 0.09,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.076,
"rtt_max": 0.256,
"rtt_avg": 0.091,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.076,
"rtt_max": 0.333,
"rtt_avg": 0.092,
"pkt_loss": 0.0
}
},
"aes256-1400": {
"iperf": {
"throughput": 840253358.805319,
"cpu_sender": 3.11095,
"cpu_receiver": 63.676868
},
"ping_100": {
"rtt_min": 0.077,
"rtt_max": 1.619,
"rtt_avg": 0.094,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.078,
"rtt_max": 0.437,
"rtt_avg": 0.096,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.083,
"rtt_max": 3.592,
"rtt_avg": 0.098,
"pkt_loss": 0.0
}
},
"chacha20-1400": {
"iperf": {
"throughput": 675882455.008626,
"cpu_sender": 2.813117,
"cpu_receiver": 60.831832
},
"ping_100": {
"rtt_min": 0.079,
"rtt_max": 0.249,
"rtt_avg": 0.096,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.085,
"rtt_max": 0.29,
"rtt_avg": 0.101,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.089,
"rtt_max": 0.222,
"rtt_avg": 0.108,
"pkt_loss": 0.0
}
},
"plain-7000": {
"iperf": {
"throughput": 4207272000.0,
"cpu_sender": 11.512448,
"cpu_receiver": 64.622926
},
"ping_100": {
"rtt_min": 0.074,
"rtt_max": 0.214,
"rtt_avg": 0.092,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.076,
"rtt_max": 0.341,
"rtt_avg": 0.091,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.079,
"rtt_max": 0.271,
"rtt_avg": 0.095,
"pkt_loss": 0.0
}
},
"aes256-7000": {
"iperf": {
"throughput": 2896202000.0,
"cpu_sender": 8.098468,
"cpu_receiver": 57.364265
},
"ping_100": {
"rtt_min": 0.078,
"rtt_max": 0.173,
"rtt_avg": 0.094,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.079,
"rtt_max": 0.424,
"rtt_avg": 0.095,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.083,
"rtt_max": 0.224,
"rtt_avg": 0.098,
"pkt_loss": 0.0
}
},
"chacha20-7000": {
"iperf": {
"throughput": 2262884000.0,
"cpu_sender": 5.687549,
"cpu_receiver": 54.00455
},
"ping_100": {
"rtt_min": 0.078,
"rtt_max": 1.712,
"rtt_avg": 0.097,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.084,
"rtt_max": 0.353,
"rtt_avg": 0.101,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.091,
"rtt_max": 0.227,
"rtt_avg": 0.107,
"pkt_loss": 0.0
}
},
"results": {
"throughput_mbits": {
"native": 9678.121,
"plain-1400": 937.787147728354,
"aes256-1400": 840.253358805319,
"chacha20-1400": 675.882455008626,
"plain-7000": 4207.272,
"aes256-7000": 2896.202,
"chacha20-7000": 2262.884
},
"latency_ms": {
"plain-1400": {
"100": 18.0,
"500": 18.0,
"1000": 18.0
},
"aes256-1400": {
"100": 20.0,
"500": 20.5,
"1000": 21.0
},
"chacha20-1400": {
"100": 21.0,
"500": 23.000000000000004,
"1000": 26.0
},
"plain-7000": {
"100": 19.0,
"500": 18.0,
"1000": 19.5
},
"aes256-7000": {
"100": 20.0,
"500": 20.0,
"1000": 21.0
},
"chacha20-7000": {
"100": 21.5,
"500": 23.000000000000004,
"1000": 25.5
}
}
}
}

View File

@ -0,0 +1,227 @@
{
"meta": {
"region": "eu-central-1",
"instance_type": "m5.large",
"ami": "ami-0a02ee601d742e89f",
"version": "1.4.0",
"duration": 869.958389043808
},
"native": {
"iperf": {
"throughput": 9680326000.0,
"cpu_sender": 16.947174,
"cpu_receiver": 62.788147
},
"ping_100": {
"rtt_min": 0.031,
"rtt_max": 0.404,
"rtt_avg": 0.035,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.033,
"rtt_max": 0.34,
"rtt_avg": 0.037,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.034,
"rtt_max": 0.216,
"rtt_avg": 0.038,
"pkt_loss": 0.0
}
},
"plain-1400": {
"iperf": {
"throughput": 940748265.414369,
"cpu_sender": 2.128748,
"cpu_receiver": 67.266424
},
"ping_100": {
"rtt_min": 0.061,
"rtt_max": 0.214,
"rtt_avg": 0.073,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.061,
"rtt_max": 0.209,
"rtt_avg": 0.074,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.063,
"rtt_max": 0.212,
"rtt_avg": 0.076,
"pkt_loss": 0.0
}
},
"aes256-1400": {
"iperf": {
"throughput": 848072373.681968,
"cpu_sender": 3.673014,
"cpu_receiver": 67.785447
},
"ping_100": {
"rtt_min": 0.062,
"rtt_max": 1.592,
"rtt_avg": 0.076,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.065,
"rtt_max": 0.276,
"rtt_avg": 0.078,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.066,
"rtt_max": 3.276,
"rtt_avg": 0.08,
"pkt_loss": 0.0
}
},
"chacha20-1400": {
"iperf": {
"throughput": 651477352.697557,
"cpu_sender": 3.299836,
"cpu_receiver": 62.990183
},
"ping_100": {
"rtt_min": 0.066,
"rtt_max": 2.066,
"rtt_avg": 0.079,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.071,
"rtt_max": 0.185,
"rtt_avg": 0.084,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.075,
"rtt_max": 0.624,
"rtt_avg": 0.089,
"pkt_loss": 0.0
}
},
"plain-7000": {
"iperf": {
"throughput": 4369926000.0,
"cpu_sender": 13.042179,
"cpu_receiver": 68.669349
},
"ping_100": {
"rtt_min": 0.06,
"rtt_max": 0.227,
"rtt_avg": 0.075,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.064,
"rtt_max": 0.22,
"rtt_avg": 0.078,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.063,
"rtt_max": 0.168,
"rtt_avg": 0.076,
"pkt_loss": 0.0
}
},
"aes256-7000": {
"iperf": {
"throughput": 3108569000.0,
"cpu_sender": 5.808026,
"cpu_receiver": 62.726234
},
"ping_100": {
"rtt_min": 0.064,
"rtt_max": 0.267,
"rtt_avg": 0.078,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.066,
"rtt_max": 0.297,
"rtt_avg": 0.08,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.069,
"rtt_max": 0.208,
"rtt_avg": 0.082,
"pkt_loss": 0.0
}
},
"chacha20-7000": {
"iperf": {
"throughput": 2136825000.0,
"cpu_sender": 4.381177,
"cpu_receiver": 50.957336
},
"ping_100": {
"rtt_min": 0.065,
"rtt_max": 0.235,
"rtt_avg": 0.08,
"pkt_loss": 0.0
},
"ping_500": {
"rtt_min": 0.072,
"rtt_max": 0.26,
"rtt_avg": 0.087,
"pkt_loss": 0.0
},
"ping_1000": {
"rtt_min": 0.078,
"rtt_max": 0.308,
"rtt_avg": 0.091,
"pkt_loss": 0.0
}
},
"results": {
"throughput_mbits": {
"native": 9680.326,
"plain-1400": 940.748265414369,
"aes256-1400": 848.072373681968,
"chacha20-1400": 651.477352697557,
"plain-7000": 4369.926,
"aes256-7000": 3108.569,
"chacha20-7000": 2136.825
},
"latency_ms": {
"plain-1400": {
"100": 18.999999999999996,
"500": 18.5,
"1000": 19.0
},
"aes256-1400": {
"100": 20.499999999999996,
"500": 20.5,
"1000": 21.0
},
"chacha20-1400": {
"100": 22.0,
"500": 23.500000000000004,
"1000": 25.5
},
"plain-7000": {
"100": 19.999999999999996,
"500": 20.5,
"1000": 19.0
},
"aes256-7000": {
"100": 21.5,
"500": 21.5,
"1000": 22.000000000000004
},
"chacha20-7000": {
"100": 22.5,
"500": 24.999999999999996,
"1000": 26.5
}
}
}
}

387
perf/perf.py Executable file
View File

@ -0,0 +1,387 @@
#!/usr/bin/env python3
import boto3
import atexit
import paramiko
import io
import time
import threading
import re
import json
import base64
import sys
from datetime import date
# Note: this script will run for 15 minutes and incur costs of about $ 0.03
REGION = "eu-central-1"
AMI = "ami-0a02ee601d742e89f"
USERNAME = "ec2-user"
INSTANCE_TYPE = "m5.large"
SPOT = True
MAX_PRICE = "0.08" # USD per hour per VM
VERSION = "1.4.0"
USERDATA = """#cloud-config
packages:
- iperf3
runcmd:
- wget https://github.com/dswd/vpncloud/releases/download/v{version}/vpncloud_{version}.x86_64.rpm -O /tmp/vpncloud.rpm
- yum install -y /tmp/vpncloud.rpm
""".format(version=VERSION)
MAX_WAIT = 300
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def run_cmd(connection, cmd):
_stdin, stdout, stderr = connection.exec_command(cmd)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
code = stdout.channel.recv_exit_status()
if code:
raise Exception("Command failed", code, out, err)
else:
return out, err
class EC2Environment:
def __init__(self):
self.vpc = None
self.igw = None
self.rtb = None
self.subnet = None
self.sg = None
self.key_pair = None
self.rsa_key = None
self.placement_group = None
self.sender = None
self.receiver = None
self.sender_request = None
self.receiver_request = None
self.sender_ssh = None
self.receiver_ssh = None
try:
self.setup()
self.wait_until_ready()
eprint("Setup done")
except:
eprint("Error, shutting down")
self.terminate()
raise
def setup(self):
ec2 = boto3.resource('ec2', region_name=REGION)
ec2client = boto3.client('ec2', region_name=REGION)
self.vpc = ec2.create_vpc(CidrBlock='172.16.0.0/16')
eprint("Created VPC {}".format(self.vpc.id))
self.vpc.create_tags(Tags=[{"Key": "Name", "Value": "vpncloud-perf-test"}])
self.vpc.wait_until_available()
ec2client.modify_vpc_attribute(VpcId=self.vpc.id, EnableDnsSupport={'Value': True})
ec2client.modify_vpc_attribute(VpcId=self.vpc.id, EnableDnsHostnames={'Value': True})
self.igw = ec2.create_internet_gateway()
eprint("Created Internet Gateway {}".format(self.igw.id))
self.igw.attach_to_vpc(VpcId=self.vpc.id)
self.rtb = self.vpc.create_route_table()
eprint("Created Routing table {}".format(self.rtb.id))
self.rtb.create_route(DestinationCidrBlock='0.0.0.0/0', GatewayId=self.igw.id)
self.subnet = ec2.create_subnet(CidrBlock='172.16.1.0/24', VpcId=self.vpc.id)
eprint("Created Subnet {}".format(self.subnet.id))
self.rtb.associate_with_subnet(SubnetId=self.subnet.id)
self.sg = ec2.create_security_group(GroupName='SSH-ONLY', Description='only allow SSH traffic', VpcId=self.vpc.id)
eprint("Created security group {}".format(self.sg.id))
self.sg.authorize_ingress(CidrIp='0.0.0.0/0', IpProtocol='tcp', FromPort=22, ToPort=22)
self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='icmp', FromPort=-1, ToPort=-1)
self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='tcp', FromPort=0, ToPort=65535)
self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='udp', FromPort=0, ToPort=65535)
self.key_pair = ec2.create_key_pair(KeyName='vpncloud-perf-test-keypair')
eprint("Created key pair {}".format(self.key_pair.name))
self.rsa_key = paramiko.RSAKey.from_private_key(io.StringIO(self.key_pair.key_material))
self.placement_group = ec2.create_placement_group(GroupName="vpncloud-test-placement", Strategy="cluster")
eprint("Created placement group {}".format(self.placement_group.name))
if SPOT:
response = ec2client.request_spot_instances(
SpotPrice = MAX_PRICE,
Type = "one-time",
InstanceCount = 2,
LaunchSpecification = {
"ImageId": AMI,
"InstanceType": INSTANCE_TYPE,
"KeyName": self.key_pair.name,
"UserData": base64.b64encode(USERDATA.encode("ascii")).decode('ascii'),
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"DeleteOnTermination": True,
"VolumeType": "gp2",
"VolumeSize": 8,
}
}
],
"NetworkInterfaces": [
{
'SubnetId': self.subnet.id,
'DeviceIndex': 0,
'AssociatePublicIpAddress': True,
'Groups': [self.sg.group_id]
}
],
"Placement": {
'GroupName': self.placement_group.name
}
}
)
sender, receiver = response['SpotInstanceRequests']
self.sender_request = sender['SpotInstanceRequestId']
self.receiver_request = receiver['SpotInstanceRequestId']
eprint("Created spot instance requests {} and {}".format(self.sender_request, self.receiver_request))
eprint("Waiting for spot instance requests")
waited = 0
while waited < MAX_WAIT:
time.sleep(1.0)
response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[self.sender_request])
sender = response['SpotInstanceRequests'][0]
response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[self.receiver_request])
receiver = response['SpotInstanceRequests'][0]
if 'InstanceId' in sender:
self.sender = ec2.Instance(sender['InstanceId'])
if 'InstanceId' in receiver:
self.receiver = ec2.Instance(receiver['InstanceId'])
if self.sender and self.receiver:
break
if waited >= MAX_WAIT:
raise Exception("Waited too long")
else:
self.sender, self.receiver = ec2.create_instances(
ImageId=AMI,
InstanceType=INSTANCE_TYPE,
MaxCount=2,
MinCount=2,
NetworkInterfaces=[
{
'SubnetId': self.subnet.id,
'DeviceIndex': 0,
'AssociatePublicIpAddress': True,
'Groups': [self.sg.group_id]
}
],
Placement={
'GroupName': self.placement_group.name
},
UserData=USERDATA,
KeyName='vpncloud-perf-test-keypair'
)
eprint("Created EC2 instances {} and {}".format(self.sender.id, self.receiver.id))
eprint("Waiting for instances to start...")
self.sender.wait_until_running()
self.receiver.wait_until_running()
self.sender.reload()
self.receiver.reload()
def wait_until_ready(self):
waited = 0
eprint("Waiting for SSH to be ready...")
while waited < MAX_WAIT:
try:
if not self.sender_ssh:
self.sender_ssh = self._connect(self.sender)
if not self.receiver_ssh:
self.receiver_ssh = self._connect(self.receiver)
break
except:
pass
time.sleep(1.0)
waited += 1
eprint("Waiting for instances to finish setup...")
while waited < MAX_WAIT:
try:
run_cmd(self.sender_ssh, 'test -f /var/lib/cloud/instance/boot-finished')
run_cmd(self.receiver_ssh, 'test -f /var/lib/cloud/instance/boot-finished')
break
except:
pass
time.sleep(1.0)
waited += 1
if waited >= MAX_WAIT:
raise Exception("Waited too long")
def terminate(self):
eprint("Deleting resources...")
if self.sender_ssh:
self.sender_ssh.close()
if self.receiver_ssh:
self.receiver_ssh.close()
if self.sender:
eprint(self.sender.id)
self.sender.terminate()
if self.receiver:
eprint(self.receiver.id)
self.receiver.terminate()
if self.sender:
self.sender.wait_until_terminated()
if self.receiver:
self.receiver.wait_until_terminated()
if self.sender_request or self.receiver_request:
ec2client = boto3.client('ec2', region_name=REGION)
if self.sender_request:
eprint(self.sender_request)
ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.sender_request])
if self.receiver_request:
eprint(self.receiver_request)
ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.receiver_request])
if self.placement_group:
self.placement_group.delete()
if self.key_pair:
eprint(self.key_pair.name)
self.key_pair.delete()
if self.sg:
eprint(self.sg.id)
self.sg.delete()
if self.subnet:
eprint(self.subnet.id)
self.subnet.delete()
if self.rtb:
eprint(self.rtb.id)
self.rtb.delete()
if self.igw:
eprint(self.igw.id)
self.igw.detach_from_vpc(VpcId=self.vpc.id)
self.igw.delete()
if self.vpc:
eprint(self.vpc.id)
self.vpc.delete()
def _connect(self, instance):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname=instance.public_dns_name, username=USERNAME, pkey=self.rsa_key, timeout=1.0, banner_timeout=1.0)
return client
class PerfTest:
def __init__(self, sender_ssh, sender_ip, receiver_ssh, receiver_ip):
self.sender_ssh = sender_ssh
self.sender_ip = sender_ip
self.receiver_ssh = receiver_ssh
self.receiver_ip = receiver_ip
self.sender_ip_vpncloud = "10.0.0.1"
self.receiver_ip_vpncloud = "10.0.0.2"
@classmethod
def from_ec2_env(cls, env):
return cls(env.sender_ssh, env.sender.private_ip_address, env.receiver_ssh, env.receiver.private_ip_address)
def run_sender(self, cmd):
return run_cmd(self.sender_ssh, cmd)
def run_receiver(self, cmd):
return run_cmd(self.receiver_ssh, cmd)
def run_ping(self, dst, size):
eprint("Running ping {} with size {} ...".format(dst, size))
(out, _) = self.run_sender('sudo ping {dst} -c 30000 -i 0.001 -s {size} -U -q'.format(dst=dst, size=size))
match = re.search(r'([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)', out)
ping_min = float(match.group(1))
ping_avg = float(match.group(2))
ping_max = float(match.group(3))
match = re.search(r'(\d*)% packet loss', out)
pkt_loss = float(match.group(1))
return {
"rtt_min": ping_min,
"rtt_max": ping_max,
"rtt_avg": ping_avg,
"pkt_loss": pkt_loss
}
def run_iperf(self, dst):
eprint("Running iperf on {} ...".format(dst))
self.run_receiver('iperf3 -s -D')
time.sleep(0.1)
(out, _) = self.run_sender('iperf3 -c {dst} -t 30 --json'.format(dst=dst))
self.run_receiver('killall iperf3')
data = json.loads(out)
return {
"throughput": data['end']['streams'][0]['receiver']['bits_per_second'],
"cpu_sender": data['end']['cpu_utilization_percent']['host_total'],
"cpu_receiver": data['end']['cpu_utilization_percent']['remote_total']
}
def run_suite(self, dst):
return {
"iperf": self.run_iperf(dst),
"ping_100": self.run_ping(dst, 100),
"ping_500": self.run_ping(dst, 500),
"ping_1000": self.run_ping(dst, 1000),
}
def start_vpncloud(self, mtu=1400, crypto=None):
eprint("Setting up vpncloud on receiver")
crypto_str = " --shared-key test --crypto {}".format(crypto) if crypto else ""
args = "-t tap --daemon -l 3210 --no-port-forwarding" + crypto_str
self.run_receiver("sudo vpncloud {args} --ifup 'ifconfig $IFNAME {ip}/24 mtu {mtu} up'".format(args=args, mtu=mtu, ip=self.receiver_ip_vpncloud))
eprint("Setting up vpncloud on sender")
self.run_sender("sudo vpncloud {args} -c {peer}:3210 --ifup 'ifconfig $IFNAME {ip}/24 mtu {mtu} up'".format(args=args, mtu=mtu, ip=self.sender_ip_vpncloud, peer=self.receiver_ip))
time.sleep(1.0)
def stop_vpncloud(self):
self.run_sender("sudo killall vpncloud")
self.run_receiver("sudo killall vpncloud")
time.sleep(3.0)
def run(self):
results = {
"meta": {
"region": REGION,
"instance_type": INSTANCE_TYPE,
"ami": AMI,
"version": VERSION
},
"native": self.run_suite(self.receiver_ip)
}
for mtu in [1400, 7000]:
for crypto in [None, "aes256", "chacha20"]:
eprint("Running with mtu {} and crypto {}".format(mtu, crypto or "plain"))
self.start_vpncloud(mtu=mtu, crypto=crypto)
res = self.run_suite(self.receiver_ip_vpncloud)
self.stop_vpncloud()
results["{}-{}".format(crypto or "plain", mtu)] = res
results['results'] = {
"throughput_mbits": dict([
(k, results[k]["iperf"]["throughput"] / 1000000.0) for k in ["native", "plain-1400", "aes256-1400", "chacha20-1400", "plain-7000", "aes256-7000", "chacha20-7000"]
]),
"latency_ms": dict([
(k, dict([
(str(s), (results[k]["ping_%s" % s]["rtt_avg"] - results["native"]["ping_%s" % s]["rtt_avg"])*1000.0/2.0) for s in [100, 500, 1000]
])) for k in ["plain-1400", "aes256-1400", "chacha20-1400", "plain-7000", "aes256-7000", "chacha20-7000"]
])
}
return results
env = EC2Environment()
atexit.register(lambda: env.terminate())
perf = PerfTest.from_ec2_env(env)
start = time.time()
results = perf.run()
duration = time.time() - start
results["meta"]["duration"] = duration
name = "{date}_{version}_perf.json".format(date=date.today().strftime('%Y-%m-%d'), version=VERSION)
eprint('Storing results in {}'.format(name))
with open(name, 'w') as fp:
json.dump(results, fp, indent=2)
eprint("done.")