Merge pull request #3 from schade/feature/Elastic_IP_enhanced

Feature/elastic ip enhanced

CHANGELOG.md

@@ -1,3 +1,17 @@
+# v4.7.1
+* Cleaning up omnibus usage to link embedded bin, ruby into default $PATHs, rather than use /etc/environment to try tweaking (doesn't hit a large number of programs)
+* Launched machines should announce their state as "started"
+
+# v4.7.0:
+(@nickmarden rocks the house again)
+* Added support for "prepare" phase, prior to any machine-specific actions
+* Move security group creation and authorization assurance to prepare phase (fixes #189)
+* Allow user/group-style security group references (fixes #207)
+* Move keypair creation to prepare phase
+
+# v4.6.2:
+* Added a -f/--with-facet option to knife cluster list
+
 # v4.6.1:
 * Fixes nested array bug when computing list of AZs for an ELB (thanks @nickmarden)
 * Cleaning up overzealous Elastic IP inclusion (alternative fix to #222, thanks @nickmarden)

VERSION

@@ -1 +1 @@
-4.6.1
+4.7.1

ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.6.1"
+  s.version = "4.7.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-12-11"
+  s.date = "2012-12-20"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "[email protected]"
   s.extra_rdoc_files = [

lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb

@@ -44,15 +44,12 @@ if [ ! -f /opt/chef/bin/chef-client ]; then
   curl -L http://www.opscode.com/chef/install.sh | sudo bash
 fi
 
-# Include the omnibus' path in the system-wide path, to allow use of
-#   its executables (gem, ruby, bundler, etc.)
-(
-cat <<'EOP'
-PATH="/opt/chef/embedded/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"
-EOP
-) >> /etc/environment
-. /etc/environment
+# Replace /usr/local/sbin with the omnibus' bin, to get it into the PATH
+mv /usr/local/sbin{,~}
+ln -s /opt/chef/embedded/bin /usr/local/sbin
 
+# Link ruby into /usr/bin/ruby, to allow /usr/bin/env 
+ln -s /opt/chef/embedded/bin/ruby /usr/bin
 
 gem install extlib bundler json right_aws pry fog
 

lib/chef/knife/cluster_kill.rb

@@ -75,6 +75,10 @@ def confirm_execution(target)
         confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
       end
 
+      def prepares?
+        false
+      end
+
     end
   end
 end

lib/chef/knife/cluster_launch.rb

@@ -82,6 +82,11 @@ def run
         section("Syncing to chef")
         target.save :providers => :chef
 
+        unless target.empty?
+          ui.info "Preparing shared resources:"
+          all_computers(*@name_args).prepare
+        end
+
         # Launch computers
         ui.info("")
         section("Launching computers", :green)

lib/chef/knife/cluster_list.rb

@@ -27,23 +27,31 @@ class ClusterList < Knife
         require 'formatador'
       end
 
-      banner "knife cluster list (options)"
-
+      banner 'knife cluster list (options)'
+
+      option :facets,
+        :long        => '--with-facets',
+        :short       => '-f',
+        :description => 'List cluster facets along with names and paths',
+        :default     => false,
+        :boolean     => true
+
       def run
         load_ironfan
         configure_dry_run
 
-        hash = Ironfan.cluster_filenames
-
-        table = []
-        hash.keys.sort.each do |key|
-          table.push( { :cluster => key, :path => hash[key] } )
+        data = Ironfan.cluster_filenames.map do |name, path|
+          as_table = { :cluster => name, :path => path }
+          if config[:facets]
+            facets = Ironfan.load_cluster(name).facets.to_a.map(&:name).join(', ')
+            as_table.merge!(:facets => facets)
+          end
+          as_table
         end
 
         ui.info "Cluster Path: #{ Ironfan.cluster_path.join ", " }"
-
-        Formatador.display_compact_table(table, [:cluster,:path])
-
+        headers = config[:facets] ? [:cluster, :facets, :path] : [:cluster, :path] 
+        Formatador.display_compact_table(data, headers)
       end
     end
   end

lib/chef/knife/cluster_proxy.rb

@@ -119,6 +119,10 @@ def proxy_pac_contents
 }\n}
       end
 
+      def prepares?
+        false
+      end
+
       def aggregates?
         false
       end

lib/chef/knife/cluster_sync.rb

@@ -62,6 +62,10 @@ def perform_execution(target)
         else Chef::Log.debug("Skipping sync to cloud") ; end
       end
 
+      def prepares_on_noop?
+        true
+      end
+
       def aggregates_on_noop?
         true
       end

lib/chef/knife/ironfan_script.rb

@@ -44,6 +44,11 @@ def run
 
       target = get_relevant_slice(* @name_args)
 
+      if prepares? and (prepares_on_noop? or not target.empty?)
+        ui.info "Preparing shared resources:"
+        all_computers(*@name_args).prepare
+      end
+
       unless target.empty?
         ui.info(["\n",
                  ui.color("Running #{sub_command}", :cyan),
@@ -76,6 +81,14 @@ def perform_execution(target)
       target.send(sub_command)
     end
 
+    def prepares?
+      true
+    end
+
+    def prepares_on_noop?
+      false
+    end
+
     def aggregates?
       true
     end

lib/ironfan/broker/computer.rb

@@ -75,6 +75,7 @@ def kill(options={})
       def launch
         ensure_dependencies
         iaas_provider.machine_class.create! self
+        node.announce_state :started
         save
         self
       end
@@ -298,11 +299,19 @@ def validate
         values.map {|c| c.providers.values}.flatten.uniq.each {|p| p.validate computers }
       end
 
-      def aggregate
+      def group_action(verb)
         computers = self
-        provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas})}.flatten.uniq
+        provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas })}.flatten.uniq
         providers     = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.compact.uniq
-        providers.each { |p| p.aggregate! computers }
+        providers.each { |p| p.send(verb, computers) }
+      end
+
+      def prepare
+        group_action(:prepare!)
+      end
+
+      def aggregate
+        group_action(:aggregate!)
       end
 
       #

lib/ironfan/dsl/ec2.rb

@@ -26,6 +26,7 @@ class Ec2 < Cloud
       magic :placement_group,           String
       magic :provider,                  Whatever,       :default => Ironfan::Provider::Ec2
       magic :elastic_ip,                String
+      magic :allocation_id,             String
       magic :region,                    String,         :default => ->{ default_region }
       collection :security_groups,      Ironfan::Dsl::Ec2::SecurityGroup, :key_method => :name
       magic :ssh_user,                  String,         :default => ->{ image_info[:ssh_user] }

lib/ironfan/provider.rb

@@ -42,6 +42,12 @@ def self.validate(computers)
       resources.each {|r| r.validate_resources! computers }
     end
 
+    def self.prepare!(computers)
+      resources.each do |r|
+        r.prepare!(computers) if r.shared?
+      end
+    end
+
     def self.aggregate!(computers)
       resources.each do |r|
         r.aggregate!(computers) if r.shared?
@@ -87,14 +93,15 @@ def on_correlate(*p)              Ironfan.noop(self,__method__,*p);   end
       #
       def self.create!(*p)              Ironfan.noop(self,__method__,*p);   end
       def self.save!(*p)                Ironfan.noop(self,__method__,*p);   end
+      def self.prepare!(*p)             Ironfan.noop(self,__method__,*p);   end
       def self.aggregate!(*p)           Ironfan.noop(self,__method__,*p);   end
       def self.destroy!(*p)             Ironfan.noop(self,__method__,*p);   end
 
       #
       # Utilities
       #
-      [:shared?, :multiple?, :load!,:validate_computer!,
-       :validate_resources!,:create!,:save!,:aggregate!,:destroy!].each do |method_name|
+      [:shared?, :multiple?, :load!,:validate_computer!, :validate_resources!,
+       :create!, :save!, :prepare!, :aggregate!, :destroy!].each do |method_name|
          define_method(method_name) {|*p| self.class.send(method_name,*p) }
        end
 

lib/ironfan/provider/ec2/elastic_ip.rb

@@ -4,7 +4,7 @@ class Ec2
 
       class ElasticIp < Ironfan::Provider::Resource
       	delegate :addresses, :associate_address, :allocation_id, 
-            :allocation_id=, :destroy, :domain, :domain=, 
+            :allocation_id=, :allocate_address, :destroy, :domain, :domain=, 
             :describe_addresses, :disassociate_address, :domain, :id,
             :network_interface_id, :network_interface_id=, :public_ip, 
             :public_ip=, :public_ip_address, :save, :server=, :server, :server_id, 
@@ -23,39 +23,87 @@ def name()                      adaptee.public_ip ;                 end
 
         def self.load!(cluster=nil)
           Ec2.connection.addresses.each do |eip|
+            pp eip
             register eip
             Chef::Log.debug("Loaded #{eip}")
 
             # The rest of this definition shows relevant information when -VV
             #   is passed to knife and aids in troubleshooting any refusal to
             #   attach Elastic IPs
             Chef::Log.debug( "AWS domain: #{eip.domain}" )
-            Chef::Log.debug( "available ip match: #{eip.public_ip}" )
+            if eip.public_ip.nil?
+              Chef::Log.debug( "no Elastic IPs currently allocated" )
+            else
+              Chef::Log.debug( "#{eip}" )
+              # Chef::Log.debug( "available ip match: #{eip.public_ip}" )
+            end
             Chef::Log.debug( "----------------------" )
           end
 
           cluster.servers.each do |s|
-            next if s.ec2.elastic_ip.nil?
+            next if not s.ec2.include?(:elastic_ip)
             if recall? s.ec2.elastic_ip
               Chef::Log.debug( "Cluster elastic_ip matches #{s.ec2.elastic_ip}" )
             else
               Chef::Log.debug( "No matching Elastic IP for #{s.ec2.elastic_ip}" )
             end
+            next if not s.ec2.include?(:allocation_id)
+            if recall? s.ec2.allocation_id
+              Chef::Log.debug( "Cluster Allocation ID matches #{s.ec2.allocation_id}" )
+            else
+              Chef::Log.debug( "No matching Allocation ID for #{s.ec2.allocation_id}" )
+            end
           end
 
         end
 
         #
         # Manipulation
         #
-
+Ec2.connection.associate_address( computer.machine.id, elastic_ip )
         def self.save!(computer)
-          return unless (computer.created? and not computer.server.ec2.elastic_ip.nil?)
-          elastic_ip = computer.server.ec2.elastic_ip
+          return unless computer.created?
+          # instead of just returning if the elastic_ip is blank we first test if the symbol exists and whether an actual 
+          # address exists in the collection; All three require the presence of elastic_ip in the facet definition. We
+          # also, in the absence of an elastic_ip value, can use allocation_id to attach a VPC Elastic IP.
+          return unless ( computer.server.ec2.include?(:elastic_ip) or computer.server.ec2.include?(:allocation_id) )
+              ui.warn("can only specify one of either Elastic IP or Allocation ID; not both.") if info.blank?
+            end
+            if ( computer.server.ec2.elastic_ip.nil? and !computer.server.ec2.include?(:allocation_id) )
+              # First,  :elastic_ip is set, no address is currently allocated for this connection's owner 
+              # NOTE: We cannot specifiy an address to create, but after a reload we can then load the first available.
+              if computer.server.addresses.nil?
+                Ec2.connection.allocate_address
+                load! 
+                elastic_ip = computer.server.addresses.first.public_ip
+                Chef::Log.debug( "allocating new Elastic IP address" )
+              else
+                # Second, :elastic_ip is set, has an address available to use but has no set value available in facet definition.
+                elastic_ip = computer.server.addresses.first.public_ip
+                Chef::Log.debug( "using first available Elastic IP address" )
+            elsif ( computer.server.ec2.include?(:elastic_ip) and !computer.server.ec2.include?(:allocation_id) )
+              # Third,  :elastic_ip is set, has an address available to use and has a set value available in facet definition. 
+              elastic_ip = computer.server.ec2.elastic_ip
+              Chef::Log.debug( "using requested Elastic IP address" )
+            elsif ( !computer.server.ec2.include?(:elastic_ip) and computer.server.ec2.include?(:allocation_id) )
+              #Fourth, :elastic_ip is unset but :allocation_id is given in facet definition. (this is functionaility for attaching VPC Elastic IPS)
+              allocation_id = computer.server.ec2.allocation_id
+              Chef::Log.debug( "using Elastic IP address matched to given Allocation ID" )
+            else 
+              ui.warn("can only specify one of either Elastic IP or Allocation ID; not both.")
+            end
+          end
+          return unless ( computer.server.ec2.include?(:allocation_id) and !computer.server.ec2.include(:elastic_ip)? )
+            if computer.server.ec2.include?(:allocation_id)
+          end
           Ironfan.step(computer.name, "associating Elastic IP #{elastic_ip}", :blue)
           Ironfan.unless_dry_run do
             Ironfan.safely do
-              Ec2.connection.associate_address( computer.machine.id, elastic_ip )
+              if !computer.server.ec2.include?(:allocation_id)
+                Ec2.connection.associate_address( computer.machine.id, public_ip = elastic_ip )
+              else 
+                Ec2.connection.associate_address( computer.machine.id, allocation_id = allocation_id )
+              end
             end
           end
         end

lib/ironfan/provider/ec2/keypair.rb

@@ -50,8 +50,9 @@ def receive_adaptee(obj)
         # Manipulation
         #
 
-        def self.create!(computer)
-          name = computer.server.cluster_name
+        def self.prepare!(computers)
+          return if computers.empty?
+          name = computers.values[0].server.cluster_name
           return if recall? name
           Ironfan.step(name, "creating key pair for #{name}", :blue)
           result = Ec2.connection.create_key_pair(name)