module SyncWrap::AmazonWS

Supports host provisioning in EC2 via AWS APIs, creating and attaching EBS volumes, and creating Route53 record sets.

Attributes

resolver_options[RW]

DNS Resolver options for testing Route53 (default: Use public google name servers to avoid local negative caching)

route53_default_rs_options[RW]

Default options for Route53 record set creation

Public Class Methods

new() click to toggle source
Calls superclass method
# File lib/syncwrap/amazon_ws.rb, line 34
def initialize
  @default_instance_options = {
    ebs_volumes:        0,
    ebs_volume_options: { size: 16 }, #gb
    lvm_volumes:        [ [ 1.00, '/data' ] ],
    security_groups:    [ :default ],
    instance_type:      'm1.medium',
    region:             'us-east-1'
  }
  @route53_default_rs_options = {
    ttl:  300,
    wait: true
  }

  @resolver_options = {
    nameserver: [ '8.8.8.8', '8.8.4.4' ]
  }

  super
end

Protected Instance Methods

aws_configure( json_file ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 57
def aws_configure( json_file )
  AWS.config( JSON.parse( IO.read( json_file ),
                          symbolize_names: true ) )
end
aws_create_instance( name, opts = {} ) click to toggle source

Create an instance, using name as the Name tag and assumed host name.

Options

See AWS::EC2::InstanceCollection.create with the following additions/differences:

:count

must be 1 or unspecified.

:region

Default 'us-east-1'

:security_groups

Array of Security Group names. The special :default value is replaced with a single security group with same name as the :region.

:ebs_volumes

The number of EBS volumes to create and attach to this instance.

:ebs_volume_options

A nested Hash of options, as per AWS::EC2::VolumeCollection.create with custom default :size 16 GB, and the same :availibility_zone as the instance.

:ebs_mounts

Device mounting scheme. The value :sdf_p indicates “/dev/sd”, and should be used for HVM instances. The default scheme is currently :sdh1_6 “/dev/sdh”. See EC2: Block Device Mapping

:lvm_volumes

Ignored here.

:roles

Array of role Strings or Symbols (applied as Roles tag)

# File lib/syncwrap/amazon_ws.rb, line 132
def aws_create_instance( name, opts = {} )
  opts = deep_merge_hashes( @default_instance_options, opts )
  region = opts.delete( :region )
  opts.delete( :lvm_volumes ) #unused here

  ec2 = AWS::EC2.new.regions[ region ]

  iopts = opts.dup
  iopts.delete( :ebs_volumes )
  iopts.delete( :ebs_volume_options )
  iopts.delete( :ebs_mounts )
  iopts.delete( :roles ) #-> tags
  iopts.delete( :description ) #-> tags
  iopts.delete( :tag ) #-> tags
  iopts.delete( :vpc )

  if iopts[ :count ] && iopts[ :count ] != 1
    raise ":count #{iopts[ :count ]} != 1 is not supported"
  end

  iopts[ :security_groups ].map! do |gname|
    gname = region if gname == :default
    aws_create_security_group( gname,
                               region: region, vpc: opts[ :vpc ] )
  end

  az = iopts[ :availability_zone ]
  iopts[ :availability_zone ] = az.call if az.is_a?( Proc )

  inst = ec2.instances.create( iopts )

  wait_until( "instance #{inst.id} to register" ) do
    inst.status != :pending
  end

  inst.add_tag( 'Name', value: name )

  if opts[ :roles ]
    inst.add_tag( 'Roles', value: opts[ :roles ].join(' ') )
  end

  if opts[ :description ]
    inst.add_tag( 'Description', value: opts[ :description ] )
  end

  tag = opts[ :tag ]
  if tag
    tag = tag.call if tag.is_a?( Proc )
    inst.add_tag( 'Tag', value: tag )
  end

  wait_for_running( inst )

  # FIXME: Split method
  # FIXME: Support alternative syntax, i.e
  # { ebs_volumes: [ [4, size: 48], [2, size: 8] ] }

  if opts[ :ebs_volumes ] > 0
    vopts = { availability_zone: inst.availability_zone }.
      merge( opts[ :ebs_volume_options ] )

    attachments = opts[ :ebs_volumes ].times.map do |i|
      vol = ec2.volumes.create( vopts )
      wait_until( vol.id, 0.5 ) { vol.status == :available }
      ap = if opts[ :ebs_mounts ] == :sdf_p
             "/dev/sd" + "fghijklmnop"[i]
           else
             "/dev/sdh#{i+1}"
           end
      vol.attach_to( inst, ap ) #=> Attachment
    end

    wait_until( "volumes to attach" ) do
      !( attachments.any? { |a| a.status == :attaching } )
    end
  end
  #FIXME: end

  instance_to_props( region, inst )
end
aws_create_security_group( name, opts = {} ) click to toggle source

Create a security_group given name and options. Currently this is a no-op if the security group of the given name already exists (in the given or default VPC or without, e.g. EC2-Classic). In either case the associated SecurityGroup object is returned.

Options

See AWS::EC2::SecurityGroupCollection.create with the following additions/differences:

:region

Required

:vpc

The VPC ID in which to create the group. Required if host is to be launched in a subnet of a non-default VPC.

:description

Optional text description. Set to name if unspecified.

# File lib/syncwrap/amazon_ws.rb, line 80
def aws_create_security_group( name, opts = {} )
  opts = opts.dup
  region = opts.delete( :region )
  vpc = opts[ :vpc ]
  ec2 = AWS::EC2.new.regions[ region ]

  sg = ec2.security_groups.find do |sg|
    sg.name == name && ( vpc.nil? || sg.vpc_id == vpc )
  end
  unless sg
    sg = ec2.security_groups.create( name, opts )
    # Allow ssh on the special "default" region named group
    if name == region
      sg.authorize_ingress(:tcp, 22)
    end
  end
  sg
end
aws_terminate_instance( iprops, delete_attached_storage = false, do_wait = true ) click to toggle source

Terminate an instance and wait for it to be terminated. If requested, /dev/sdh# (PV style) or /dev/sd[f-p] attached EBS volumes which are not otherwise marked for :delete_on_termination will also be terminated. The minimum required properties in iprops are :region and :id.

WARNING: data will be lost!

# File lib/syncwrap/amazon_ws.rb, line 304
def aws_terminate_instance( iprops, delete_attached_storage = false, do_wait = true )
  ec2 = AWS::EC2.new.regions[ iprops[ :region ] ]
  inst = ec2.instances[ iprops[ :id ] ]
  unless inst.exists?
    raise "Instance #{iprops[:id]} does not exist in #{iprops[:region]}"
  end

  ebs_volumes = []
  if delete_attached_storage
    ebs_volumes = inst.block_devices.map do |dev|
      ebs = dev[ :ebs ]

      if ebs && !ebs[:delete_on_termination] &&
         ( dev[:device_name] =~ /dh\d+$/ ||
           dev[:device_name] =~ /sd[f-p]$/ )
        ebs[ :volume_id ]
      end
    end.compact
  end

  inst.terminate
  if do_wait || !ebs_volumes.empty?
    wait_until( "termination of #{inst.id}", 2.0 ) { inst.status == :terminated }
  end

  ebs_volumes = ebs_volumes.map do |vid|
    volume = ec2.volumes[ vid ]
    if volume.exists?
      volume
    else
      puts "WARN: #{volume} doesn't exist"
      nil
    end
  end.compact

  ebs_volumes.each do |vol|
    wait_until( "deletion of vol #{vol.id}" ) do
      vol.status == :available || vol.status == :deleted
    end
    vol.delete if vol.status == :available
  end

end
create_image( host, opts = {} ) click to toggle source

Create an image for the specified host which will be stopped before imaging and not restarted

Options

:name

Required, image compatable (i.e. no spaces, identifier) name

:description

Image description

# File lib/syncwrap/amazon_ws.rb, line 230
def create_image( host, opts = {} )
  opts = opts.dup
  name = opts.delete( :name ) or raise "Missing required name for image"
  region = host[ :region ]
  ec2 = AWS::EC2.new.regions[ region ]
  inst = ec2.instances[ host[ :id ] ]
  raise "Host ID #{host[:id]} does not exist?" unless inst.exists?

  inst.stop
  stat = wait_until( "instance #{inst.id} to stop", 2.0 ) do
    s = inst.status
    s if s == :stopped || s == :terminated
  end
  raise "Instance #{inst.id} has status #{stat}" unless stat == :stopped

  image = inst.create_image( name, { no_reboot: true }.merge( opts ) )
  stat = wait_until( "image #{image.id} to be available", 2.0 ) do
    s = image.state
    s if s != :pending
  end
  unless stat == :available
    raise "Image #{image.id} failed: #{image.state_reason}"
  end

  image.image_id
end
decode_roles( roles ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 386
def decode_roles( roles )
  ( roles || "" ).split( /\s+/ ).map( &:to_sym )
end
deep_merge_hashes( h1, h2 ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 408
def deep_merge_hashes( h1, h2 )
  h1.merge( h2 ) do |key, v1, v2|
    if v1.is_a?( Hash ) && v2.is_a?( Hash )
      deep_merge_hashes( v1, v2 )
    else
      v2
    end
  end
end
dot_terminate( name ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 404
def dot_terminate( name )
  ( name =~ /\.$/ ) ? name : ( name + '.' )
end
image_name_exist?( region, name ) click to toggle source

Return true if the authenticated AWS user in the sepecified region already owns an image of the specified name

# File lib/syncwrap/amazon_ws.rb, line 215
def image_name_exist?( region, name )
  ec2 = AWS::EC2.new.regions[ region ]
  images = ec2.images.with_owner( :self )
  images.any? { |img| img.name == name }
end
import_host_props( regions ) click to toggle source

Find running or pending instances in each region String and convert to a HostList.

# File lib/syncwrap/amazon_ws.rb, line 357
def import_host_props( regions )
  regions.inject([]) do |insts, region|
    ec2 = AWS::EC2.new.regions[ region ]

    found = ec2.instances.map do |inst|
      next unless [ :running, :pending ].include?( inst.status )
      instance_to_props( region, inst )
    end

    insts + found.compact
  end

end
instance_to_props( region, inst ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 371
def instance_to_props( region, inst )
  tags = inst.tags.to_h

  { id:      inst.id,
    region:  region,
    availability_zone: inst.availability_zone,
    ami:     inst.image_id,
    name:    tags[ 'Name' ],
    internet_name:  inst.dns_name,
    internet_ip:    inst.ip_address,
    internal_ip:    inst.private_ip_address,
    instance_type:  inst.instance_type,
    roles:   decode_roles( tags[ 'Roles' ] ) }
end
route53_create_host_cname( iprops, opts = {} ) click to toggle source

Create a Route53 DNS CNAME from iprops :name to :internet_name. Options are per AWS::Route53::ResourceRecordSetCollection.create (currently undocumented) with the following additions:

:domain_name

name of the hosted zone and suffix for CNAME. Should terminate in a DOT '.'

:wait

If true, wait for CNAME to resolve

# File lib/syncwrap/amazon_ws.rb, line 264
def route53_create_host_cname( iprops, opts = {} )
  opts = deep_merge_hashes( @route53_default_rs_options, opts )
  dname = dot_terminate( opts.delete( :domain_name ) )
  do_wait = opts.delete( :wait )
  rs_opts = opts.
    merge( resource_records: [ {value: iprops[:internet_name]} ] )

  r53 = AWS::Route53.new
  zone = r53.hosted_zones.find { |hz| hz.name == dname } or
    raise "Route53 Hosted Zone name #{dname} not found"
  long_name = [ iprops[:name], dname ].join('.')
  zone.rrsets.create( long_name, 'CNAME', rs_opts )
  wait_for_dns_resolve( long_name, dname ) if do_wait
end
wait_for_dns_resolve( long_name, domain, rtype = Resolv::DNS::Resource::IN::CNAME ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 279
def wait_for_dns_resolve( long_name,
                          domain,
                          rtype = Resolv::DNS::Resource::IN::CNAME )

  ns_addr = Resolv::DNS.open( @resolver_options ) do |rvr|
    ns_n = rvr.getresource( domain, Resolv::DNS::Resource::IN::SOA ).mname
    rvr.getaddress( ns_n ).to_s
  end

  sleep 3 # Initial wait

  wait_until( "#{long_name} to resolve", 3.0 ) do
    Resolv::DNS.open( nameserver: ns_addr ) do |rvr|
      rvr.getresources( long_name, rtype ).first
    end
  end
end
wait_for_running( inst ) click to toggle source
# File lib/syncwrap/amazon_ws.rb, line 348
def wait_for_running( inst )
  wait_until( "instance #{inst.id} to run", 2.0 ) { inst.status != :pending }
  stat = inst.status
  raise "Instance #{inst.id} has status #{stat}" unless stat == :running
  nil
end
wait_until( desc, freq = 1.0 ) click to toggle source

Wait until block returns truthy, sleeping for freq seconds between attempts. Writes desc and a sequence of DOTs on a single line until complete.

# File lib/syncwrap/amazon_ws.rb, line 393
def wait_until( desc, freq = 1.0 )
  $stdout.write( "Waiting for " + desc )
  until (ret = yield) do
    $stdout.write '.'
    sleep freq
  end
  ret
ensure
  puts
end