Restructure ec2 test framework

.gitignore

@@ -9,3 +9,6 @@ dist
 builder/cache
 .idea
 release.sh
+__pycache__
+*.pem
+.vscode

contrib/common.py

@@ -0,0 +1,369 @@
+import boto3
+import atexit
+import paramiko
+import io
+import time
+import threading
+import re
+import json
+import base64
+import sys
+from datetime import date
+
+MAX_WAIT = 300
+CREATE = "***CREATE***"
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def run_cmd(connection, cmd):
+    _stdin, stdout, stderr = connection.exec_command(cmd)
+    out = stdout.read().decode('utf-8')
+    err = stderr.read().decode('utf-8')
+    code = stdout.channel.recv_exit_status()
+    if code:
+        raise Exception("Command failed", code, out, err)
+    else:
+        return out, err
+
+
+class SpotInstanceRequest:
+    def __init__(self, id):
+        self.id = id
+    def __str__(self):
+        return str(self.id)
+
+
+class Node:
+    def __init__(self, instance, connection):
+        self.instance = instance
+        self.connection = connection
+        self.private_ip = instance.private_ip_address
+        self.public_ip = instance.public_ip_address
+
+    def run_cmd(self, cmd):
+        return run_cmd(self.connection, cmd)
+
+    def start_vpncloud(self, ifup=None, mtu=1400, ip=None, crypto=None, shared_key="test", device_type="tap", listen="3210", mode="normal", peers=[], subnets=[]):
+        args = [
+            "--daemon", 
+            "--no-port-forwarding", 
+            "-t {}".format(device_type),
+            "-m {}".format(mode),
+            "-l {}".format(listen)
+        ]
+        if ifup:
+            args.append("--ifup {}".format(ifup))
+        else:
+            args.append("--ifup 'ifconfig $IFNAME {ip} mtu {mtu} up'".format(mtu=mtu, ip=ip))
+        if crypto:
+            args.append("--shared-key '{}' --crypto {}".format(shared_key, crypto))
+        for p in peers:
+            args.append("-c {}".format(p))
+        for s in subnets:
+            args.append("-s {}".format(s))
+        args = " ".join(args)
+        self.run_cmd("sudo vpncloud {}".format(args))
+
+    def stop_vpncloud(self, wait=True):
+        self.run_cmd("sudo killall vpncloud")
+        if wait:
+            time.sleep(3.0)
+
+    def ping(self, dst, size=100, count=10, interval=0.001):
+        (out, _) = self.run_cmd('sudo ping {dst} -c {count} -i {interval} -s {size} -U -q'.format(dst=dst, size=size, count=count, interval=interval))
+        match = re.search(r'([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)', out)
+        ping_min = float(match.group(1))
+        ping_avg = float(match.group(2))
+        ping_max = float(match.group(3))
+        match = re.search(r'(\d*)% packet loss', out)
+        pkt_loss = float(match.group(1))
+        return {
+            "rtt_min": ping_min,
+            "rtt_max": ping_max,
+            "rtt_avg": ping_avg,
+            "pkt_loss": pkt_loss
+        }
+
+    def start_iperf_server(self):
+        self.run_cmd('iperf3 -s -D')
+        time.sleep(0.1)
+
+    def stop_iperf_server(self):
+        self.run_cmd('killall iperf3')
+
+    def run_iperf(self, dst, duration):
+        (out, _) = self.run_cmd('iperf3 -c {dst} -t {duration} --json'.format(dst=dst, duration=duration))
+        data = json.loads(out)
+        return {
+            "throughput": data['end']['streams'][0]['receiver']['bits_per_second'],
+            "cpu_sender": data['end']['cpu_utilization_percent']['host_total'],
+            "cpu_receiver": data['end']['cpu_utilization_percent']['remote_total']
+        }
+    
+
+def find_ami(region, owner, name_pattern, arch='x86_64'):
+    ec2client = boto3.client('ec2', region_name=region)
+    response = ec2client.describe_images(Owners=[owner], Filters=[
+        {'Name': 'name', 'Values': [name_pattern]}, 
+        {'Name': 'architecture', 'Values': ['x86_64']}
+    ])
+    try:
+        image = max(response['Images'], key=lambda i: i['CreationDate'])
+        return image['ImageId']
+    except ValueError:
+        return None
+
+
+class EC2Environment:
+    def __init__(self, vpncloud_version, region, node_count, instance_type, use_spot=True, max_price=0.1, ami=('amazon', 'amzn2-ami-hvm-*'), username="ec2-user", subnet=CREATE, keyname=CREATE, privatekey=CREATE, tag="vpncloud", cluster_nodes=False):
+        self.region = region
+        self.node_count = node_count
+        self.instance_type = instance_type
+        self.use_spot = use_spot
+        self.max_price = str(max_price)
+        if isinstance(ami, tuple):
+            owner, name = ami
+            self.ami = find_ami(region, owner, name)
+            assert self.ami
+        else:
+            self.ami = ami
+        self.username = username
+        self.vpncloud_version = vpncloud_version
+        self.cluster_nodes = cluster_nodes
+        self.resources = []
+        self.instances = []
+        self.connections = []
+        self.nodes = []
+        self.subnet = subnet
+        self.tag = tag
+        self.keyname = keyname
+        self.privatekey = privatekey
+        self.rsa_key = None
+        try:
+            eprint("Setting up resources...")
+            self.setup()
+            self.wait_until_ready()
+            for i in range(0, self.node_count):
+                self.nodes.append(Node(self.instances[i], self.connections[i]))
+            eprint("Setup done")
+            atexit.register(lambda : self.terminate())
+            eprint()
+        except:
+            eprint("Error, shutting down")
+            self.terminate()
+            raise
+
+    def track_resource(self, res):
+        self.resources.append(res)
+        eprint("\t{} {}".format(res.__class__.__name__, res.id if hasattr(res, "id") else ""))
+        if hasattr(res, "create_tags") and not hasattr(res, "name"):
+            res.create_tags(Tags=[{"Key": "Name", "Value": self.tag}])
+
+    def setup_vpc(self):
+        ec2 = boto3.resource('ec2', region_name=self.region)
+        ec2client = boto3.client('ec2', region_name=self.region)
+
+        vpc = ec2.create_vpc(CidrBlock='172.16.0.0/16')
+        self.track_resource(vpc)
+        vpc.wait_until_available()
+        ec2client.modify_vpc_attribute(VpcId=vpc.id, EnableDnsSupport={'Value': True})
+        ec2client.modify_vpc_attribute(VpcId=vpc.id, EnableDnsHostnames={'Value': True})
+
+        igw = ec2.create_internet_gateway()
+        self.track_resource(igw)
+        igw.attach_to_vpc(VpcId=vpc.id)
+
+        rtb = vpc.create_route_table()
+        self.track_resource(rtb)
+        rtb.create_route(DestinationCidrBlock='0.0.0.0/0', GatewayId=igw.id)
+
+        subnet = ec2.create_subnet(CidrBlock='172.16.1.0/24', VpcId=vpc.id)
+        self.track_resource(subnet)
+        rtb.associate_with_subnet(SubnetId=subnet.id)
+
+        self.subnet = subnet.id
+
+
+    def setup(self):
+        ec2 = boto3.resource('ec2', region_name=self.region)
+        ec2client = boto3.client('ec2', region_name=self.region)
+
+        if self.subnet == CREATE:
+            self.setup_vpc()
+        else:
+            eprint("\tUsing subnet {}".format(self.subnet))
+
+        vpc = ec2.Subnet(self.subnet).vpc
+
+        sg = ec2.create_security_group(GroupName='SSH-ONLY', Description='only allow SSH traffic', VpcId=vpc.id)
+        self.track_resource(sg)
+        sg.authorize_ingress(CidrIp='0.0.0.0/0', IpProtocol='tcp', FromPort=22, ToPort=22)
+        sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='icmp', FromPort=-1, ToPort=-1)
+        sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='tcp', FromPort=0, ToPort=65535)
+        sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='udp', FromPort=0, ToPort=65535)
+
+        if self.keyname == CREATE:
+            key_pair = ec2.create_key_pair(KeyName="{}-keypair".format(self.tag))
+            self.track_resource(key_pair)
+            self.keyname = key_pair.name
+            self.privatekey = key_pair.key_material
+        self.rsa_key = paramiko.RSAKey.from_private_key(io.StringIO(self.privatekey))
+
+        placement = {}
+        if self.cluster_nodes:
+            placement_group = ec2.create_placement_group(GroupName="{}-placement".format(self.tag), Strategy="cluster")
+            self.track_resource(placement_group)
+            placement = { 'GroupName': placement_group.name }
+        
+        userdata = """#cloud-config
+packages:
+  - iperf3
+runcmd:
+  - wget https://github.com/dswd/vpncloud/releases/download/v{version}/vpncloud_{version}.x86_64.rpm -O /tmp/vpncloud.rpm
+  - yum install -y /tmp/vpncloud.rpm
+""".format(version=self.vpncloud_version)
+        
+        if self.use_spot:
+            response = ec2client.request_spot_instances(
+                SpotPrice = self.max_price,
+                Type = "one-time",
+                InstanceCount = self.node_count,
+                LaunchSpecification = {
+                    "ImageId": self.ami,
+                    "InstanceType": self.instance_type,
+                    "KeyName": key_pair.name,
+                    "UserData": base64.b64encode(userdata.encode("ascii")).decode('ascii'),
+                    "BlockDeviceMappings": [
+                        {
+                            "DeviceName": "/dev/xvda",
+                            "Ebs": {
+                                "DeleteOnTermination": True,
+                                "VolumeType": "gp2",
+                                "VolumeSize": 8,
+                            }
+                        }
+                    ],
+                    "NetworkInterfaces": [
+                        {
+                            'SubnetId': self.subnet,
+                            'DeviceIndex': 0,
+                            'AssociatePublicIpAddress': True,
+                            'Groups': [sg.group_id]
+                        }
+                    ],
+                    "Placement": placement
+                }
+            )
+            requests = []
+            for req in response['SpotInstanceRequests']:
+                request = SpotInstanceRequest(req['SpotInstanceRequestId'])
+                self.track_resource(request)
+                requests.append(request)
+            eprint("Waiting for spot instance requests")
+            waited = 0
+            self.instances = [None] * len(requests)
+            while waited < MAX_WAIT:
+                time.sleep(1.0)
+                for i, req in enumerate(requests):
+                    response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[req.id])
+                    data = response['SpotInstanceRequests'][0]
+                    if 'InstanceId' in data:
+                        self.instances[i] = ec2.Instance(data['InstanceId'])
+                        self.track_resource(self.instances[i])
+                if min(map(bool, self.instances)):
+                    break
+            if waited >= MAX_WAIT:
+                raise Exception("Waited too long")
+        else:
+            self.instances = ec2.create_instances(
+                ImageId=self.ami,
+                InstanceType=self.instance_type,
+                MaxCount=self.node_count,
+                MinCount=self.node_count,
+                NetworkInterfaces=[
+                    {
+                        'SubnetId': self.subnet.id,
+                        'DeviceIndex': 0,
+                        'AssociatePublicIpAddress': True,
+                        'Groups': [sg.group_id]
+                    }
+                ],
+                Placement=placement,
+                UserData=userdata,
+                KeyName='vpncloud-perf-test-keypair'
+            )
+            for instance in self.instances:
+                self.track_resource(instance)
+
+    def wait_until_ready(self):
+        waited = 0
+        eprint("Waiting for instances to start...")
+        for instance in self.instances:
+            instance.wait_until_running()
+            instance.reload()
+        eprint("Waiting for SSH to be ready...")
+        self.connections = [None] * len(self.instances)
+        while waited < MAX_WAIT:
+            for i, instance in enumerate(self.instances):
+                if self.connections[i]:
+                    continue
+                try:
+                    self.connections[i] = self._connect(instance)
+                except:
+                    pass
+            if min(map(bool, self.connections)):
+                break
+            time.sleep(1.0)
+            waited += 1
+        eprint("Waiting for instances to finish setup...")
+        ready = [False] * len(self.connections)
+        while waited < MAX_WAIT:
+            for i, con in enumerate(self.connections):
+                if ready[i]:
+                    continue
+                try:
+                    run_cmd(con, 'test -f /var/lib/cloud/instance/boot-finished')
+                    ready[i] = True
+                except:
+                    pass
+            if min(map(bool, ready)):
+                break
+            time.sleep(1.0)
+            waited += 1
+        if waited >= MAX_WAIT:
+            raise Exception("Waited too long")
+
+    def terminate(self):
+        if not self.resources:
+            return
+        eprint("Closing connections...")
+        for con in self.connections:
+            if con:
+                con.close()
+        self.connections = []
+        eprint("Terminating instances...")
+        for instance in self.instances:
+            instance.terminate()
+        for instance in self.instances:
+            eprint("\t{}".format(instance.id))
+            instance.wait_until_terminated()
+        self.instances = []
+        eprint("Deleting resources...")
+        ec2client = boto3.client('ec2', region_name=self.region)
+        for res in reversed(self.resources):
+            eprint("\t{} {}".format(res.__class__.__name__, res.id if hasattr(res, "id") else ""))
+            if isinstance(res, SpotInstanceRequest):
+                ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[res.id])
+            if hasattr(res, "attachments"):
+                for a in res.attachments:
+                    res.detach_from_vpc(VpcId=a['VpcId'])
+            if hasattr(res, "delete"):
+                res.delete()
+        self.resources = []
+
+    def _connect(self, instance):
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        client.connect(hostname=instance.public_dns_name, username=self.username, pkey=self.rsa_key, timeout=1.0, banner_timeout=1.0)
+        return client

contrib/example.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from common import EC2Environment, CREATE
+import time
+
+setup = EC2Environment(
+    region = "eu-central-1", 
+    node_count = 2, 
+    instance_type = 't3.nano', 
+    vpncloud_version = "1.4.0"
+)
+
+sender = setup.nodes[0]
+receiver = setup.nodes[1]
+
+sender.start_vpncloud(ip="10.0.0.1/24")
+receiver.start_vpncloud(ip="10.0.0.2/24", peers=["{}:3210".format(sender.private_ip)])
+time.sleep(1.0)
+
+sender.ping("10.0.0.2")
+
+sender.stop_vpncloud()
+receiver.stop_vpncloud()

contrib/performance.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+
+from common import EC2Environment, CREATE, eprint
+import time, json
+from datetime import date
+
+
+# Note: this script will run for ~8 minutes and incur costs of about $ 0.02
+
+VERSION = "1.4.0"
+REGION = "eu-central-1"
+
+env = EC2Environment(
+    region = REGION, 
+    node_count = 2, 
+    instance_type = "m5.large", 
+    use_spot = True, 
+    max_price = "0.08", # USD per hour per VM
+    vpncloud_version = VERSION, 
+    cluster_nodes = True,
+    subnet = CREATE, 
+    keyname = CREATE
+)
+
+
+CRYPTO = ["aes256", "aes128", "chacha20"] if VERSION >= "1.5.0" else ["aes256", "chacha20"]
+
+
+class PerfTest:
+    def __init__(self, sender, receiver, meta):
+        self.sender = sender
+        self.receiver = receiver
+        self.sender_ip_vpncloud = "10.0.0.1"
+        self.receiver_ip_vpncloud = "10.0.0.2"
+        self.meta = meta
+
+    @classmethod
+    def from_ec2_env(cls, env):
+        meta = {
+            "region": env.region,
+            "instance_type": env.instance_type,
+            "ami": env.ami,
+            "version": env.version
+        }
+        return cls(env.nodes[0], env.nodes[1], meta)
+
+    def run_ping(self, dst, size):
+        eprint("\tRunning ping {} with size {} ...".format(dst, size))
+        return self.sender.ping(dst=dst, size=size, count=30000, interval=0.001)
+
+    def run_iperf(self, dst):
+        eprint("\tRunning iperf on {} ...".format(dst))
+        self.receiver.start_iperf_server()
+        time.sleep(0.1)
+        result = self.sender.run_iperf(dst=dst, duration=30)
+        self.receiver.stop_iperf_server()
+        return result
+
+    def run_suite(self, dst):
+        return {
+            "iperf": self.run_iperf(dst),
+            "ping_100": self.run_ping(dst, 100),
+            "ping_500": self.run_ping(dst, 500),
+            "ping_1000": self.run_ping(dst, 1000),
+        }
+
+    def start_vpncloud(self, mtu=8800, crypto=None):
+        eprint("\tSetting up vpncloud on receiver")
+        self.receiver.start_vpncloud(crypto=crypto, ip="{}/24".format(self.receiver_ip_vpncloud), mtu=mtu)
+        eprint("\tSetting up vpncloud on sender")
+        self.sender.start_vpncloud(crypto=crypto, peers=["{}:3210".format(self.receiver.private_ip)], ip="{}/24".format(self.sender_ip_vpncloud), mtu=mtu)
+        time.sleep(1.0)
+
+    def stop_vpncloud(self):
+        self.sender.stop_vpncloud(wait=False)
+        self.receiver.stop_vpncloud(wait=True)
+
+    def run(self):
+        eprint("Testing native network")
+        results = {
+            "meta": self.meta,
+            "native": self.run_suite(self.receiver.private_ip)
+        }
+        for crypto in [None] + CRYPTO:
+            eprint("Running with crypto {}".format(crypto or "plain"))
+            self.start_vpncloud(crypto=crypto)
+            res = self.run_suite(self.receiver_ip_vpncloud)
+            self.stop_vpncloud()
+            results[str(crypto or "plain")] = res
+        results['results'] = {
+            "throughput_mbits": dict([
+                (k, results[k]["iperf"]["throughput"] / 1000000.0) for k in ["native", "plain"] + CRYPTO
+            ]),
+            "latency_us": dict([
+                (k, dict([
+                    (str(s), (results[k]["ping_%s" % s]["rtt_avg"] - results["native"]["ping_%s" % s]["rtt_avg"])*1000.0/2.0) for s in [100, 500, 1000]
+                ])) for k in ["plain"] + CRYPTO
+            ])
+        }
+        return results
+
+perf = PerfTest.from_ec2_env(env)
+
+start = time.time()
+results = perf.run()
+duration = time.time() - start
+
+results["meta"]["duration"] = duration
+
+name = "measurements/{date}_{version}_perf.json".format(date=date.today().strftime('%Y-%m-%d'), version=VERSION)
+eprint('Storing results in {}'.format(name))
+with open(name, 'w') as fp:
+    json.dump(results, fp, indent=2)
+eprint("done.")

contrib/testnet.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+from common import EC2Environment, CREATE
+import atexit, argparse, os
+
+REGION = "eu-central-1"
+
+VERSION = "1.4.0"
+
+
+parser = argparse.ArgumentParser(description='Create a test setup')
+parser.add_argument('--instancetype', default='t3.nano', help='EC2 instance type')
+parser.add_argument('--version', default=VERSION, help='VpnCloud version to use')
+parser.add_argument('--count', '-c', dest="count", type=int, default=2, help='Number of instance to create')
+parser.add_argument('--cluster', action="store_true", help='Cluster instances to get reliable throughput')
+parser.add_argument('--subnet', help='AWS subnet id to use (empty = create new one)')
+parser.add_argument('--keyname', help='Name of AWS keypair to use (empty = create new one)')
+parser.add_argument('--keyfile', help='Path of the private key file')
+
+
+args = parser.parse_args()
+
+privatekey = None
+if args.keyname:
+    with open(args.keyfile, 'r') as fp:
+        privatekey = fp.read()
+
+setup = EC2Environment(
+    region = REGION, 
+    node_count = args.count, 
+    instance_type = args.instancetype, 
+    vpncloud_version = args.version, 
+    cluster_nodes = args.cluster,
+    subnet = args.subnet or CREATE, 
+    keyname = args.keyname or CREATE,
+    privatekey = privatekey
+)
+
+if not args.keyname:
+    assert not os.path.exists(args.keyfile)
+    with open(args.keyfile, 'x') as fp:
+        fp.write(setup.privatekey)
+    os.chmod(args.keyfile, 0o400)
+    print("SSH private key written to {}".format(args.keyfile))
+    atexit.register(lambda : os.remove(args.keyfile))
+    print()
+
+print("Nodes:")
+for node in setup.nodes:
+    print("\t {}@{}\tprivate: {}".format(setup.username, node.public_ip, node.private_ip))
+print()
+
+print("Press ENTER to shut down")
+input()

perf/perf.py

@@ -1,390 +0,0 @@
-#!/usr/bin/env python3
-
-import boto3
-import atexit
-import paramiko
-import io
-import time
-import threading
-import re
-import json
-import base64
-import sys
-from datetime import date
-
-
-# Note: this script will run for ~8 minutes and incur costs of about $ 0.02
-
-REGION = "eu-central-1"
-AMI = "ami-0a02ee601d742e89f"
-USERNAME = "ec2-user"
-INSTANCE_TYPE = "m5.large"
-SPOT = True
-MAX_PRICE = "0.08" # USD per hour per VM
-
-VERSION = "1.4.0"
-
-USERDATA = """#cloud-config
-packages:
-  - iperf3
-runcmd:
-  - wget https://github.com/dswd/vpncloud/releases/download/v{version}/vpncloud_{version}.x86_64.rpm -O /tmp/vpncloud.rpm
-  - yum install -y /tmp/vpncloud.rpm
-""".format(version=VERSION)
-
-MAX_WAIT = 300
-
-CRYPTO = ["aes256", "aes128", "chacha20"] if VERSION >= "1.5.0" else ["aes256", "chacha20"]
-
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-def run_cmd(connection, cmd):
-    _stdin, stdout, stderr = connection.exec_command(cmd)
-    out = stdout.read().decode('utf-8')
-    err = stderr.read().decode('utf-8')
-    code = stdout.channel.recv_exit_status()
-    if code:
-        raise Exception("Command failed", code, out, err)
-    else:
-        return out, err
-
-
-class EC2Environment:
-    def __init__(self):
-        self.vpc = None
-        self.igw = None
-        self.rtb = None
-        self.subnet = None
-        self.sg = None
-        self.key_pair = None
-        self.rsa_key = None
-        self.placement_group = None
-        self.sender = None
-        self.receiver = None
-        self.sender_request = None
-        self.receiver_request = None
-        self.sender_ssh = None
-        self.receiver_ssh = None
-        try:
-            eprint("Setting up resources...")
-            self.setup()
-            self.wait_until_ready()
-            eprint("Setup done")
-        except:
-            eprint("Error, shutting down")
-            self.terminate()
-            raise
-
-    def setup(self):
-        ec2 = boto3.resource('ec2', region_name=REGION)
-        ec2client = boto3.client('ec2', region_name=REGION)
-
-        self.vpc = ec2.create_vpc(CidrBlock='172.16.0.0/16')
-        eprint("\tCreated VPC {}".format(self.vpc.id))
-        self.vpc.create_tags(Tags=[{"Key": "Name", "Value": "vpncloud-perf-test"}])
-        self.vpc.wait_until_available()
-        ec2client.modify_vpc_attribute(VpcId=self.vpc.id, EnableDnsSupport={'Value': True})
-        ec2client.modify_vpc_attribute(VpcId=self.vpc.id, EnableDnsHostnames={'Value': True})
-
-        self.igw = ec2.create_internet_gateway()
-        eprint("\tCreated Internet Gateway {}".format(self.igw.id))
-        self.igw.attach_to_vpc(VpcId=self.vpc.id)
-
-        self.rtb = self.vpc.create_route_table()
-        eprint("\tCreated Routing table {}".format(self.rtb.id))
-        self.rtb.create_route(DestinationCidrBlock='0.0.0.0/0', GatewayId=self.igw.id)
-
-        self.subnet = ec2.create_subnet(CidrBlock='172.16.1.0/24', VpcId=self.vpc.id)
-        eprint("\tCreated Subnet {}".format(self.subnet.id))
-        self.rtb.associate_with_subnet(SubnetId=self.subnet.id)
-
-        self.sg = ec2.create_security_group(GroupName='SSH-ONLY', Description='only allow SSH traffic', VpcId=self.vpc.id)
-        eprint("\tCreated security group {}".format(self.sg.id))
-        self.sg.authorize_ingress(CidrIp='0.0.0.0/0', IpProtocol='tcp', FromPort=22, ToPort=22)
-        self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='icmp', FromPort=-1, ToPort=-1)
-        self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='tcp', FromPort=0, ToPort=65535)
-        self.sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='udp', FromPort=0, ToPort=65535)
-
-        self.key_pair = ec2.create_key_pair(KeyName='vpncloud-perf-test-keypair')
-        eprint("\tCreated key pair {}".format(self.key_pair.name))
-        self.rsa_key = paramiko.RSAKey.from_private_key(io.StringIO(self.key_pair.key_material))
-        self.placement_group = ec2.create_placement_group(GroupName="vpncloud-test-placement", Strategy="cluster")
-        eprint("\tCreated placement group {}".format(self.placement_group.name))
-        if SPOT:
-            response = ec2client.request_spot_instances(
-                SpotPrice = MAX_PRICE,
-                Type = "one-time",
-                InstanceCount = 2,
-                LaunchSpecification = {
-                    "ImageId": AMI,
-                    "InstanceType": INSTANCE_TYPE,
-                    "KeyName": self.key_pair.name,
-                    "UserData": base64.b64encode(USERDATA.encode("ascii")).decode('ascii'),
-                    "BlockDeviceMappings": [
-                        {
-                            "DeviceName": "/dev/xvda",
-                            "Ebs": {
-                                "DeleteOnTermination": True,
-                                "VolumeType": "gp2",
-                                "VolumeSize": 8,
-                            }
-                        }
-                    ],
-                    "NetworkInterfaces": [
-                        {
-                            'SubnetId': self.subnet.id,
-                            'DeviceIndex': 0,
-                            'AssociatePublicIpAddress': True,
-                            'Groups': [self.sg.group_id]
-                        }
-                    ],
-                    "Placement": {
-                        'GroupName': self.placement_group.name
-                    }
-                }
-            )
-            sender, receiver = response['SpotInstanceRequests']
-            self.sender_request = sender['SpotInstanceRequestId']
-            self.receiver_request = receiver['SpotInstanceRequestId']
-            eprint("\tCreated spot instance requests {} and {}".format(self.sender_request, self.receiver_request))
-            eprint("\tWaiting for spot instance requests")
-            waited = 0
-            while waited < MAX_WAIT:
-                time.sleep(1.0)
-                response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[self.sender_request])
-                sender = response['SpotInstanceRequests'][0]
-                response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[self.receiver_request])
-                receiver = response['SpotInstanceRequests'][0]
-                if 'InstanceId' in sender:
-                    self.sender = ec2.Instance(sender['InstanceId'])
-                if 'InstanceId' in receiver:
-                    self.receiver = ec2.Instance(receiver['InstanceId'])
-                if self.sender and self.receiver:
-                    break
-            if waited >= MAX_WAIT:
-                raise Exception("Waited too long")
-        else:
-            self.sender, self.receiver = ec2.create_instances(
-                ImageId=AMI,
-                InstanceType=INSTANCE_TYPE,
-                MaxCount=2,
-                MinCount=2,
-                NetworkInterfaces=[
-                    {
-                        'SubnetId': self.subnet.id,
-                        'DeviceIndex': 0,
-                        'AssociatePublicIpAddress': True,
-                        'Groups': [self.sg.group_id]
-                    }
-                ],
-                Placement={
-                    'GroupName': self.placement_group.name
-                },
-                UserData=USERDATA,
-                KeyName='vpncloud-perf-test-keypair'
-            )
-        eprint("\tCreated EC2 instances {} and {}".format(self.sender.id, self.receiver.id))
-        eprint("\tWaiting for instances to start...")
-        self.sender.wait_until_running()
-        self.receiver.wait_until_running()
-        self.sender.reload()
-        self.receiver.reload()
-
-    def wait_until_ready(self):
-        waited = 0
-        eprint("\tWaiting for SSH to be ready...")
-        while waited < MAX_WAIT:
-            try:
-                if not self.sender_ssh:
-                    self.sender_ssh = self._connect(self.sender)
-                if not self.receiver_ssh:
-                    self.receiver_ssh = self._connect(self.receiver)
-                break
-            except:
-                pass
-            time.sleep(1.0)
-            waited += 1
-        eprint("\tWaiting for instances to finish setup...")
-        while waited < MAX_WAIT:
-            try:
-                run_cmd(self.sender_ssh, 'test -f /var/lib/cloud/instance/boot-finished')
-                run_cmd(self.receiver_ssh, 'test -f /var/lib/cloud/instance/boot-finished')
-                break
-            except:
-                pass
-            time.sleep(1.0)
-            waited += 1
-        if waited >= MAX_WAIT:
-            raise Exception("Waited too long")
-
-    def terminate(self):
-        eprint("Deleting resources...")
-        if self.sender_ssh:
-            self.sender_ssh.close()
-        if self.receiver_ssh:
-            self.receiver_ssh.close()
-        if self.sender:
-            eprint(self.sender.id)
-            self.sender.terminate()
-        if self.receiver:
-            eprint(self.receiver.id)
-            self.receiver.terminate()
-        if self.sender:
-            self.sender.wait_until_terminated()
-        if self.receiver:
-            self.receiver.wait_until_terminated()
-        if self.sender_request or self.receiver_request:
-            ec2client = boto3.client('ec2', region_name=REGION)
-        if self.sender_request:
-            eprint(self.sender_request)
-            ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.sender_request])
-        if self.receiver_request:
-            eprint(self.receiver_request)
-            ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.receiver_request])
-        if self.placement_group:
-            self.placement_group.delete()
-        if self.key_pair:
-            eprint(self.key_pair.name)
-            self.key_pair.delete()
-        if self.sg:
-            eprint(self.sg.id)
-            self.sg.delete()
-        if self.subnet:
-            eprint(self.subnet.id)
-            self.subnet.delete()
-        if self.rtb:
-            eprint(self.rtb.id)
-            self.rtb.delete()
-        if self.igw:
-            eprint(self.igw.id)
-            self.igw.detach_from_vpc(VpcId=self.vpc.id)
-            self.igw.delete()
-        if self.vpc:
-            eprint(self.vpc.id)
-            self.vpc.delete()
-
-    def _connect(self, instance):
-        client = paramiko.SSHClient()
-        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        client.connect(hostname=instance.public_dns_name, username=USERNAME, pkey=self.rsa_key, timeout=1.0, banner_timeout=1.0)
-        return client
-
-
-class PerfTest:
-    def __init__(self, sender_ssh, sender_ip, receiver_ssh, receiver_ip):
-        self.sender_ssh = sender_ssh
-        self.sender_ip = sender_ip
-        self.receiver_ssh = receiver_ssh
-        self.receiver_ip = receiver_ip
-        self.sender_ip_vpncloud = "10.0.0.1"
-        self.receiver_ip_vpncloud = "10.0.0.2"
-
-    @classmethod
-    def from_ec2_env(cls, env):
-        return cls(env.sender_ssh, env.sender.private_ip_address, env.receiver_ssh, env.receiver.private_ip_address)
-
-    def run_sender(self, cmd):
-        return run_cmd(self.sender_ssh, cmd)
-
-    def run_receiver(self, cmd):
-        return run_cmd(self.receiver_ssh, cmd)
-
-    def run_ping(self, dst, size):
-        eprint("\tRunning ping {} with size {} ...".format(dst, size))
-        (out, _) = self.run_sender('sudo ping {dst} -c 30000 -i 0.001 -s {size} -U -q'.format(dst=dst, size=size))
-        match = re.search(r'([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)', out)
-        ping_min = float(match.group(1))
-        ping_avg = float(match.group(2))
-        ping_max = float(match.group(3))
-        match = re.search(r'(\d*)% packet loss', out)
-        pkt_loss = float(match.group(1))
-        return {
-            "rtt_min": ping_min,
-            "rtt_max": ping_max,
-            "rtt_avg": ping_avg,
-            "pkt_loss": pkt_loss
-        }
-
-    def run_iperf(self, dst):
-        eprint("\tRunning iperf on {} ...".format(dst))
-        self.run_receiver('iperf3 -s -D')
-        time.sleep(0.1)
-        (out, _) = self.run_sender('iperf3 -c {dst} -t 30 --json'.format(dst=dst))
-        self.run_receiver('killall iperf3')
-        data = json.loads(out)
-        return {
-            "throughput": data['end']['streams'][0]['receiver']['bits_per_second'],
-            "cpu_sender": data['end']['cpu_utilization_percent']['host_total'],
-            "cpu_receiver": data['end']['cpu_utilization_percent']['remote_total']
-        }
-
-    def run_suite(self, dst):
-        return {
-            "iperf": self.run_iperf(dst),
-            "ping_100": self.run_ping(dst, 100),
-            "ping_500": self.run_ping(dst, 500),
-            "ping_1000": self.run_ping(dst, 1000),
-        }
-
-    def start_vpncloud(self, mtu=8800, crypto=None):
-        eprint("\tSetting up vpncloud on receiver")
-        crypto_str = " --shared-key test --crypto {}".format(crypto) if crypto else ""
-        args = "-t tap --daemon -l 3210 --no-port-forwarding" + crypto_str
-        self.run_receiver("sudo vpncloud {args} --ifup 'ifconfig $IFNAME {ip}/24 mtu {mtu} up'".format(args=args, mtu=mtu, ip=self.receiver_ip_vpncloud))
-        eprint("\tSetting up vpncloud on sender")
-        self.run_sender("sudo vpncloud {args} -c {peer}:3210 --ifup 'ifconfig $IFNAME {ip}/24 mtu {mtu} up'".format(args=args, mtu=mtu, ip=self.sender_ip_vpncloud, peer=self.receiver_ip))
-        time.sleep(1.0)
-
-    def stop_vpncloud(self):
-        self.run_sender("sudo killall vpncloud")
-        self.run_receiver("sudo killall vpncloud")
-        time.sleep(3.0)
-
-    def run(self):
-        eprint("Testing native network")
-        results = {
-            "meta": {
-                "region": REGION,
-                "instance_type": INSTANCE_TYPE,
-                "ami": AMI,
-                "version": VERSION
-            },
-            "native": self.run_suite(self.receiver_ip)
-        }
-        for crypto in [None] + CRYPTO:
-            eprint("Running with crypto {}".format(crypto or "plain"))
-            self.start_vpncloud(mtu=8800, crypto=crypto)
-            res = self.run_suite(self.receiver_ip_vpncloud)
-            self.stop_vpncloud()
-            results[str(crypto or "plain")] = res
-        results['results'] = {
-            "throughput_mbits": dict([
-                (k, results[k]["iperf"]["throughput"] / 1000000.0) for k in ["native", "plain"] + CRYPTO
-            ]),
-            "latency_us": dict([
-                (k, dict([
-                    (str(s), (results[k]["ping_%s" % s]["rtt_avg"] - results["native"]["ping_%s" % s]["rtt_avg"])*1000.0/2.0) for s in [100, 500, 1000]
-                ])) for k in ["plain"] + CRYPTO
-            ])
-        }
-        return results
-
-
-
-env = EC2Environment()
-atexit.register(lambda: env.terminate())
-
-perf = PerfTest.from_ec2_env(env)
-
-start = time.time()
-results = perf.run()
-duration = time.time() - start
-
-results["meta"]["duration"] = duration
-
-name = "{date}_{version}_perf.json".format(date=date.today().strftime('%Y-%m-%d'), version=VERSION)
-eprint('Storing results in {}'.format(name))
-with open(name, 'w') as fp:
-    json.dump(results, fp, indent=2)
-eprint("done.")