vpncloud/contrib/common.py

import boto3
import atexit
import paramiko
import io
import time
import threading
import re
import json
import base64
import sys
from datetime import date

MAX_WAIT = 300
CREATE = "***CREATE***"

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def run_cmd(connection, cmd):
    _stdin, stdout, stderr = connection.exec_command(cmd)
    out = stdout.read().decode('utf-8')
    err = stderr.read().decode('utf-8')
    code = stdout.channel.recv_exit_status()
    if code:
        raise Exception("Command failed", code, out, err)
    else:
        return out, err


class SpotInstanceRequest:
    def __init__(self, id):
        self.id = id
    def __str__(self):
        return str(self.id)


class Node:
    def __init__(self, instance, connection):
        self.instance = instance
        self.connection = connection
        self.private_ip = instance.private_ip_address
        self.public_ip = instance.public_ip_address

    def run_cmd(self, cmd):
        return run_cmd(self.connection, cmd)

    def start_vpncloud(self, ifup=None, mtu=1400, ip=None, crypto=None, shared_key="test", device_type="tap", listen="3210", mode="normal", peers=[], subnets=[]):
        args = [
            "--daemon", 
            "--no-port-forwarding", 
            "-t {}".format(device_type),
            "-m {}".format(mode),
            "-l {}".format(listen)
        ]
        if ifup:
            args.append("--ifup {}".format(ifup))
        else:
            args.append("--ifup 'ifconfig $IFNAME {ip} mtu {mtu} up'".format(mtu=mtu, ip=ip))
        if crypto:
            args.append("--shared-key '{}' --crypto {}".format(shared_key, crypto))
        for p in peers:
            args.append("-c {}".format(p))
        for s in subnets:
            args.append("-s {}".format(s))
        args = " ".join(args)
        self.run_cmd("sudo vpncloud {}".format(args))

    def stop_vpncloud(self, wait=True):
        self.run_cmd("sudo killall vpncloud")
        if wait:
            time.sleep(3.0)

    def ping(self, dst, size=100, count=10, interval=0.001):
        (out, _) = self.run_cmd('sudo ping {dst} -c {count} -i {interval} -s {size} -U -q'.format(dst=dst, size=size, count=count, interval=interval))
        match = re.search(r'([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)/([\d]*\.[\d]*)', out)
        ping_min = float(match.group(1))
        ping_avg = float(match.group(2))
        ping_max = float(match.group(3))
        match = re.search(r'(\d*)% packet loss', out)
        pkt_loss = float(match.group(1))
        return {
            "rtt_min": ping_min,
            "rtt_max": ping_max,
            "rtt_avg": ping_avg,
            "pkt_loss": pkt_loss
        }

    def start_iperf_server(self):
        self.run_cmd('iperf3 -s -D')
        time.sleep(0.1)

    def stop_iperf_server(self):
        self.run_cmd('killall iperf3')

    def run_iperf(self, dst, duration):
        (out, _) = self.run_cmd('iperf3 -c {dst} -t {duration} --json'.format(dst=dst, duration=duration))
        data = json.loads(out)
        return {
            "throughput": data['end']['streams'][0]['receiver']['bits_per_second'],
            "cpu_sender": data['end']['cpu_utilization_percent']['host_total'],
            "cpu_receiver": data['end']['cpu_utilization_percent']['remote_total']
        }
    

def find_ami(region, owner, name_pattern, arch='x86_64'):
    ec2client = boto3.client('ec2', region_name=region)
    response = ec2client.describe_images(Owners=[owner], Filters=[
        {'Name': 'name', 'Values': [name_pattern]}, 
        {'Name': 'architecture', 'Values': ['x86_64']}
    ])
    try:
        image = max(response['Images'], key=lambda i: i['CreationDate'])
        return image['ImageId']
    except ValueError:
        return None


class EC2Environment:
    def __init__(self, vpncloud_version, region, node_count, instance_type, use_spot=True, max_price=0.1, ami=('amazon', 'amzn2-ami-hvm-*'), username="ec2-user", subnet=CREATE, keyname=CREATE, privatekey=CREATE, tag="vpncloud", cluster_nodes=False):
        self.region = region
        self.node_count = node_count
        self.instance_type = instance_type
        self.use_spot = use_spot
        self.max_price = str(max_price)
        if isinstance(ami, tuple):
            owner, name = ami
            self.ami = find_ami(region, owner, name)
            assert self.ami
        else:
            self.ami = ami
        self.username = username
        self.vpncloud_version = vpncloud_version
        self.cluster_nodes = cluster_nodes
        self.resources = []
        self.instances = []
        self.connections = []
        self.nodes = []
        self.subnet = subnet
        self.tag = tag
        self.keyname = keyname
        self.privatekey = privatekey
        self.rsa_key = None
        try:
            eprint("Setting up resources...")
            self.setup()
            self.wait_until_ready()
            for i in range(0, self.node_count):
                self.nodes.append(Node(self.instances[i], self.connections[i]))
            eprint("Setup done")
            atexit.register(lambda : self.terminate())
            eprint()
        except:
            eprint("Error, shutting down")
            self.terminate()
            raise

    def track_resource(self, res):
        self.resources.append(res)
        eprint("\t{} {}".format(res.__class__.__name__, res.id if hasattr(res, "id") else ""))
        if hasattr(res, "create_tags") and not hasattr(res, "name"):
            res.create_tags(Tags=[{"Key": "Name", "Value": self.tag}])

    def setup_vpc(self):
        ec2 = boto3.resource('ec2', region_name=self.region)
        ec2client = boto3.client('ec2', region_name=self.region)

        vpc = ec2.create_vpc(CidrBlock='172.16.0.0/16')
        self.track_resource(vpc)
        vpc.wait_until_available()
        ec2client.modify_vpc_attribute(VpcId=vpc.id, EnableDnsSupport={'Value': True})
        ec2client.modify_vpc_attribute(VpcId=vpc.id, EnableDnsHostnames={'Value': True})

        igw = ec2.create_internet_gateway()
        self.track_resource(igw)
        igw.attach_to_vpc(VpcId=vpc.id)

        rtb = vpc.create_route_table()
        self.track_resource(rtb)
        rtb.create_route(DestinationCidrBlock='0.0.0.0/0', GatewayId=igw.id)

        subnet = ec2.create_subnet(CidrBlock='172.16.1.0/24', VpcId=vpc.id)
        self.track_resource(subnet)
        rtb.associate_with_subnet(SubnetId=subnet.id)

        self.subnet = subnet.id


    def setup(self):
        ec2 = boto3.resource('ec2', region_name=self.region)
        ec2client = boto3.client('ec2', region_name=self.region)

        if self.subnet == CREATE:
            self.setup_vpc()
        else:
            eprint("\tUsing subnet {}".format(self.subnet))

        vpc = ec2.Subnet(self.subnet).vpc

        sg = ec2.create_security_group(GroupName='SSH-ONLY', Description='only allow SSH traffic', VpcId=vpc.id)
        self.track_resource(sg)
        sg.authorize_ingress(CidrIp='0.0.0.0/0', IpProtocol='tcp', FromPort=22, ToPort=22)
        sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='icmp', FromPort=-1, ToPort=-1)
        sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='tcp', FromPort=0, ToPort=65535)
        sg.authorize_ingress(CidrIp='172.16.1.0/24', IpProtocol='udp', FromPort=0, ToPort=65535)

        if self.keyname == CREATE:
            key_pair = ec2.create_key_pair(KeyName="{}-keypair".format(self.tag))
            self.track_resource(key_pair)
            self.keyname = key_pair.name
            self.privatekey = key_pair.key_material
        self.rsa_key = paramiko.RSAKey.from_private_key(io.StringIO(self.privatekey))

        placement = {}
        if self.cluster_nodes:
            placement_group = ec2.create_placement_group(GroupName="{}-placement".format(self.tag), Strategy="cluster")
            self.track_resource(placement_group)
            placement = { 'GroupName': placement_group.name }
        
        userdata = """#cloud-config
packages:
  - iperf3
runcmd:
  - wget https://github.com/dswd/vpncloud/releases/download/v{version}/vpncloud_{version}.x86_64.rpm -O /tmp/vpncloud.rpm
  - yum install -y /tmp/vpncloud.rpm
""".format(version=self.vpncloud_version)
        
        if self.use_spot:
            response = ec2client.request_spot_instances(
                SpotPrice = self.max_price,
                Type = "one-time",
                InstanceCount = self.node_count,
                LaunchSpecification = {
                    "ImageId": self.ami,
                    "InstanceType": self.instance_type,
                    "KeyName": key_pair.name,
                    "UserData": base64.b64encode(userdata.encode("ascii")).decode('ascii'),
                    "BlockDeviceMappings": [
                        {
                            "DeviceName": "/dev/xvda",
                            "Ebs": {
                                "DeleteOnTermination": True,
                                "VolumeType": "gp2",
                                "VolumeSize": 8,
                            }
                        }
                    ],
                    "NetworkInterfaces": [
                        {
                            'SubnetId': self.subnet,
                            'DeviceIndex': 0,
                            'AssociatePublicIpAddress': True,
                            'Groups': [sg.group_id]
                        }
                    ],
                    "Placement": placement
                }
            )
            requests = []
            for req in response['SpotInstanceRequests']:
                request = SpotInstanceRequest(req['SpotInstanceRequestId'])
                self.track_resource(request)
                requests.append(request)
            eprint("Waiting for spot instance requests")
            waited = 0
            self.instances = [None] * len(requests)
            while waited < MAX_WAIT:
                time.sleep(1.0)
                for i, req in enumerate(requests):
                    response = ec2client.describe_spot_instance_requests(SpotInstanceRequestIds=[req.id])
                    data = response['SpotInstanceRequests'][0]
                    if 'InstanceId' in data:
                        self.instances[i] = ec2.Instance(data['InstanceId'])
                        self.track_resource(self.instances[i])
                if min(map(bool, self.instances)):
                    break
            if waited >= MAX_WAIT:
                raise Exception("Waited too long")
        else:
            self.instances = ec2.create_instances(
                ImageId=self.ami,
                InstanceType=self.instance_type,
                MaxCount=self.node_count,
                MinCount=self.node_count,
                NetworkInterfaces=[
                    {
                        'SubnetId': self.subnet.id,
                        'DeviceIndex': 0,
                        'AssociatePublicIpAddress': True,
                        'Groups': [sg.group_id]
                    }
                ],
                Placement=placement,
                UserData=userdata,
                KeyName='vpncloud-perf-test-keypair'
            )
            for instance in self.instances:
                self.track_resource(instance)

    def wait_until_ready(self):
        waited = 0
        eprint("Waiting for instances to start...")
        for instance in self.instances:
            instance.wait_until_running()
            instance.reload()
        eprint("Waiting for SSH to be ready...")
        self.connections = [None] * len(self.instances)
        while waited < MAX_WAIT:
            for i, instance in enumerate(self.instances):
                if self.connections[i]:
                    continue
                try:
                    self.connections[i] = self._connect(instance)
                except:
                    pass
            if min(map(bool, self.connections)):
                break
            time.sleep(1.0)
            waited += 1
        eprint("Waiting for instances to finish setup...")
        ready = [False] * len(self.connections)
        while waited < MAX_WAIT:
            for i, con in enumerate(self.connections):
                if ready[i]:
                    continue
                try:
                    run_cmd(con, 'test -f /var/lib/cloud/instance/boot-finished')
                    ready[i] = True
                except:
                    pass
            if min(map(bool, ready)):
                break
            time.sleep(1.0)
            waited += 1
        if waited >= MAX_WAIT:
            raise Exception("Waited too long")

    def terminate(self):
        if not self.resources:
            return
        eprint("Closing connections...")
        for con in self.connections:
            if con:
                con.close()
        self.connections = []
        eprint("Terminating instances...")
        for instance in self.instances:
            instance.terminate()
        for instance in self.instances:
            eprint("\t{}".format(instance.id))
            instance.wait_until_terminated()
        self.instances = []
        eprint("Deleting resources...")
        ec2client = boto3.client('ec2', region_name=self.region)
        for res in reversed(self.resources):
            eprint("\t{} {}".format(res.__class__.__name__, res.id if hasattr(res, "id") else ""))
            if isinstance(res, SpotInstanceRequest):
                ec2client.cancel_spot_instance_requests(SpotInstanceRequestIds=[res.id])
            if hasattr(res, "attachments"):
                for a in res.attachments:
                    res.detach_from_vpc(VpcId=a['VpcId'])
            if hasattr(res, "delete"):
                res.delete()
        self.resources = []

    def _connect(self, instance):
        client = paramiko.SSHClient()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(hostname=instance.public_dns_name, username=self.username, pkey=self.rsa_key, timeout=1.0, banner_timeout=1.0)
        return client