#!/usr/bin/env python3 import boto3 import atexit import paramiko import io import time import threading import re import json import base64 import sys from datetime import date # Note: this script will run for ~8 minutes and incur costs of about $ 0.02 REGION = "eu-central-1" AMI = "ami-0a02ee601d742e89f" USERNAME = "ec2-user" INSTANCE_TYPE = "m5.large" SPOT = True MAX_PRICE = "0.08" # USD per hour per VM VERSION = "1.4.0" USERDATA = """#cloud-config packages: - iperf3 runcmd: - wget https://github.com/dswd/vpncloud/releases/download/v{version}/vpncloud_{version}.x86_64.rpm -O /tmp/vpncloud.rpm - yum install -y /tmp/vpncloud.rpm """.format(version=VERSION) MAX_WAIT = 300 CRYPTO = ["aes256", "aes128", "chacha20"] if VERSION >= "1.5.0" else ["aes256", "chacha20"] def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) def run_cmd(connection, cmd): _stdin, stdout, stderr = connection.exec_command(cmd) out = stdout.read().decode('utf-8') err = stderr.read().decode('utf-8') code = stdout.channel.recv_exit_status() if code: raise Exception("Command failed", code, out, err) else: return out, err class EC2Environment: def __init__(self): self.vpc = None self.igw = None self.rtb = None self.subnet = None self.sg = None self.key_pair = None self.rsa_key = None self.placement_group = None self.sender = None self.receiver = None self.sender_request = None self.receiver_request = None self.sender_ssh = None self.receiver_ssh = None try: eprint("Setting up resources...") self.setup() self.wait_until_ready() eprint("Setup done") except: eprint("Error, shutting down") self.terminate() raise def setup(self): ec2 = boto3.resource('ec2', region_name=REGION) ec2client = boto3.client('ec2', region_name=REGION) self.vpc = ec2.create_vpc(CidrBlock='172.16.0.0/16') eprint("\tCreated VPC {}".format(self.vpc.id)) self.vpc.create_tags(Tags=[{"Key": "Name", "Value": "vpncloud-perf-test"}]) self.vpc.wait_until_available() ec2client.modify_vpc_attribute(VpcId=self.vpc.id, EnableDnsSupport={'Value': True}) ec2client.modify_vpc_attribute(VpcId=self.vpc.id, EnableDnsHostnames={'Value': True}) self.igw = ec2.create_internet_gateway() eprint("\tCreated Internet Gateway {}".format(self.igw.id)) self.igw.attach_to_vpc(VpcId=self.vpc.id) self.rtb = self.vpc.create_route_table() eprint("\tCreated Routing table {}".format(self.rtb.id)) self.rtb.create_route(DestinationCidrBlock='0.0.0.0/0', GatewayId=self.igw.id) self.subnet = ec2.create_subnet(CidrBlock='172.16.1.0/24', VpcId=self.vpc.id) eprint("\tCreated Subnet {}".format(self.subnet.id)) self.rtb.associate_with_subnet(SubnetId=self.subnet.id) self.sg = ec2.create_security_group(GroupName='SSH-ONLY', Description='only allow SSH traffic', VpcId=self.vpc.id) eprint("\tCreated security group {}".format(self.sg.id)) self.sg.authorize_ingress(CidrIp='0.0.0.0/0', IpProtocol='tcp', FromPort=22, ToPort=22) self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='icmp', FromPort=-1, ToPort=-1) self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='tcp', FromPort=0, ToPort=65535) self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='udp', FromPort=0, ToPort=65535) self.key_pair = ec2.create_key_pair(KeyName='vpncloud-perf-test-keypair') eprint("\tCreated key pair {}".format(self.key_pair.name)) self.rsa_key = paramiko.RSAKey.from_private_key(io.StringIO(self.key_pair.key_material)) self.placement_group = ec2.create_placement_group(GroupName="vpncloud-test-placement", Strategy="cluster") eprint("\tCreated placement group {}".format(self.placement_group.name)) if SPOT: response = ec2client.request_spot_instances( SpotPrice = MAX_PRICE, Type = "one-time", InstanceCount = 2, LaunchSpecification = { "ImageId": AMI, "InstanceType": INSTANCE_TYPE, "KeyName": self.key_pair.name, "UserData": base64.b64encode(USERDATA.encode("ascii")).decode('ascii'), "BlockDeviceMappings": [ { "DeviceName": "/dev/xvda", "Ebs": { "DeleteOnTermination": True, "VolumeType": "gp2", "VolumeSize": 8, } } ], "NetworkInterfaces": [ { 'SubnetId': self.subnet.id, 'DeviceIndex': 0, 'AssociatePublicIpAddress': True, 'Groups': [self.sg.group_id] } ], "Placement": { 'GroupName': self.placement_group.name } } ) sender, receiver = response['SpotInstanceRequests'] self.sender_request = sender['SpotInstanceRequestId'] self.receiver_request = receiver['SpotInstanceRequestId'] eprint("\tCreated spot instance requests {} and {}".format(self.sender_request, self.receiver_request)) eprint("\tWaiting for spot instance requests") waited = 0 while waited < MAX_WAIT: time.sleep(1.0) response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[self.sender_request]) sender = response['SpotInstanceRequests'][0] response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[self.receiver_request]) receiver = response['SpotInstanceRequests'][0] if 'InstanceId' in sender: self.sender = ec2.Instance(sender['InstanceId']) if 'InstanceId' in receiver: self.receiver = ec2.Instance(receiver['InstanceId']) if self.sender and self.receiver: break if waited >= MAX_WAIT: raise Exception("Waited too long") else: self.sender, self.receiver = ec2.create_instances( ImageId=AMI, InstanceType=INSTANCE_TYPE, MaxCount=2, MinCount=2, NetworkInterfaces=[ { 'SubnetId': self.subnet.id, 'DeviceIndex': 0, 'AssociatePublicIpAddress': True, 'Groups': [self.sg.group_id] } ], Placement={ 'GroupName': self.placement_group.name }, UserData=USERDATA, KeyName='vpncloud-perf-test-keypair' ) eprint("\tCreated EC2 instances {} and {}".format(self.sender.id, self.receiver.id)) eprint("\tWaiting for instances to start...") self.sender.wait_until_running() self.receiver.wait_until_running() self.sender.reload() self.receiver.reload() def wait_until_ready(self): waited = 0 eprint("\tWaiting for SSH to be ready...") while waited < MAX_WAIT: try: if not self.sender_ssh: self.sender_ssh = self._connect(self.sender) if not self.receiver_ssh: self.receiver_ssh = self._connect(self.receiver) break except: pass time.sleep(1.0) waited += 1 eprint("\tWaiting for instances to finish setup...") while waited < MAX_WAIT: try: run_cmd(self.sender_ssh, 'test -f /var/lib/cloud/instance/boot-finished') run_cmd(self.receiver_ssh, 'test -f /var/lib/cloud/instance/boot-finished') break except: pass time.sleep(1.0) waited += 1 if waited >= MAX_WAIT: raise Exception("Waited too long") def terminate(self): eprint("Deleting resources...") if self.sender_ssh: self.sender_ssh.close() if self.receiver_ssh: self.receiver_ssh.close() if self.sender: eprint(self.sender.id) self.sender.terminate() if self.receiver: eprint(self.receiver.id) self.receiver.terminate() if self.sender: self.sender.wait_until_terminated() if self.receiver: self.receiver.wait_until_terminated() if self.sender_request or self.receiver_request: ec2client = boto3.client('ec2', region_name=REGION) if self.sender_request: eprint(self.sender_request) ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.sender_request]) if self.receiver_request: eprint(self.receiver_request) ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.receiver_request]) if self.placement_group: self.placement_group.delete() if self.key_pair: eprint(self.key_pair.name) self.key_pair.delete() if self.sg: eprint(self.sg.id) self.sg.delete() if self.subnet: eprint(self.subnet.id) self.subnet.delete() if self.rtb: eprint(self.rtb.id) self.rtb.delete() if self.igw: eprint(self.igw.id) self.igw.detach_from_vpc(VpcId=self.vpc.id) self.igw.delete() if self.vpc: eprint(self.vpc.id) self.vpc.delete() def _connect(self, instance): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostname=instance.public_dns_name, username=USERNAME, pkey=self.rsa_key, timeout=1.0, banner_timeout=1.0) return client class PerfTest: def __init__(self, sender_ssh, sender_ip, receiver_ssh, receiver_ip): self.sender_ssh = sender_ssh self.sender_ip = sender_ip self.receiver_ssh = receiver_ssh self.receiver_ip = receiver_ip self.sender_ip_vpncloud = "10.0.0.1" self.receiver_ip_vpncloud = "10.0.0.2" @classmethod def from_ec2_env(cls, env): return cls(env.sender_ssh, env.sender.private_ip_address, env.receiver_ssh, env.receiver.private_ip_address) def run_sender(self, cmd): return run_cmd(self.sender_ssh, cmd) def run_receiver(self, cmd): return run_cmd(self.receiver_ssh, cmd) def run_ping(self, dst, size): eprint("\tRunning ping {} with size {} ...".format(dst, size)) (out, _) = self.run_sender('sudo ping {dst} -c 30000 -i 0.001 -s {size} -U -q'.format(dst=dst, size=size)) match = re.search(r'([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)', out) ping_min = float(match.group(1)) ping_avg = float(match.group(2)) ping_max = float(match.group(3)) match = re.search(r'(\d*)% packet loss', out) pkt_loss = float(match.group(1)) return { "rtt_min": ping_min, "rtt_max": ping_max, "rtt_avg": ping_avg, "pkt_loss": pkt_loss } def run_iperf(self, dst): eprint("\tRunning iperf on {} ...".format(dst)) self.run_receiver('iperf3 -s -D') time.sleep(0.1) (out, _) = self.run_sender('iperf3 -c {dst} -t 30 --json'.format(dst=dst)) self.run_receiver('killall iperf3') data = json.loads(out) return { "throughput": data['end']['streams'][0]['receiver']['bits_per_second'], "cpu_sender": data['end']['cpu_utilization_percent']['host_total'], "cpu_receiver": data['end']['cpu_utilization_percent']['remote_total'] } def run_suite(self, dst): return { "iperf": self.run_iperf(dst), "ping_100": self.run_ping(dst, 100), "ping_500": self.run_ping(dst, 500), "ping_1000": self.run_ping(dst, 1000), } def start_vpncloud(self, mtu=8800, crypto=None): eprint("\tSetting up vpncloud on receiver") crypto_str = " --shared-key test --crypto {}".format(crypto) if crypto else "" args = "-t tap --daemon -l 3210 --no-port-forwarding" + crypto_str self.run_receiver("sudo vpncloud {args} --ifup 'ifconfig $IFNAME {ip}/24 mtu {mtu} up'".format(args=args, mtu=mtu, ip=self.receiver_ip_vpncloud)) eprint("\tSetting up vpncloud on sender") self.run_sender("sudo vpncloud {args} -c {peer}:3210 --ifup 'ifconfig $IFNAME {ip}/24 mtu {mtu} up'".format(args=args, mtu=mtu, ip=self.sender_ip_vpncloud, peer=self.receiver_ip)) time.sleep(1.0) def stop_vpncloud(self): self.run_sender("sudo killall vpncloud") self.run_receiver("sudo killall vpncloud") time.sleep(3.0) def run(self): eprint("Testing native network") results = { "meta": { "region": REGION, "instance_type": INSTANCE_TYPE, "ami": AMI, "version": VERSION }, "native": self.run_suite(self.receiver_ip) } for crypto in [None] + CRYPTO: eprint("Running with crypto {}".format(crypto or "plain")) self.start_vpncloud(mtu=8800, crypto=crypto) res = self.run_suite(self.receiver_ip_vpncloud) self.stop_vpncloud() results[str(crypto or "plain")] = res results['results'] = { "throughput_mbits": dict([ (k, results[k]["iperf"]["throughput"] / 1000000.0) for k in ["native", "plain"] + CRYPTO ]), "latency_us": dict([ (k, dict([ (str(s), (results[k]["ping_%s" % s]["rtt_avg"] - results["native"]["ping_%s" % s]["rtt_avg"])*1000.0/2.0) for s in [100, 500, 1000] ])) for k in ["plain"] + CRYPTO ]) } return results env = EC2Environment() atexit.register(lambda: env.terminate()) perf = PerfTest.from_ec2_env(env) start = time.time() results = perf.run() duration = time.time() - start results["meta"]["duration"] = duration name = "{date}_{version}_perf.json".format(date=date.today().strftime('%Y-%m-%d'), version=VERSION) eprint('Storing results in {}'.format(name)) with open(name, 'w') as fp: json.dump(results, fp, indent=2) eprint("done.")