Class BoxGrinder::S3Plugin
In: lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb
lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb
Parent: BasePlugin

Methods

Public Instance methods

[Source]

    # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 29
29:     def after_init
30:       register_supported_os("fedora", ['13', '14', '15', '16'])
31:       register_supported_os("centos", ['5'])
32:       register_supported_os("rhel", ['5', '6'])
33:       register_supported_os("sl", ['5', '6'])
34: 
35:       @ami_build_dir = "#{@dir.base}/ami"
36:       @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml"
37:     end

[Source]

    # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 29
29:     def after_init
30:       register_supported_os("fedora", ['13', '14', '15', '16'])
31:       register_supported_os("centos", ['5'])
32:       register_supported_os("rhel", ['5', '6'])
33:       register_supported_os("sl", ['5', '6'])
34: 
35:       @ami_build_dir = "#{@dir.base}/ami"
36:       @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml"
37:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 187
187:     def ami_by_manifest_key(ami_manifest_key)
188:       ami = @ec2.images.with_owner(@plugin_config['account_number']).
189:           filter("manifest-location","#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
190:       return nil unless ami.any?
191:       ami.first
192:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 187
187:     def ami_by_manifest_key(ami_manifest_key)
188:       ami = @ec2.images.with_owner(@plugin_config['account_number']).
189:           filter("manifest-location","#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
190:       return nil unless ami.any?
191:       ami.first
192:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 194
194:     def ami_key(appliance_name, path)
195:       base_path = "#{@s3helper.parse_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}"
196: 
197:       return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot']
198: 
199:       @log.info "Determining snapshot name"
200:       snapshot = 1
201:       while @s3helper.stub_s3obj(@bucket, "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/").exists?
202:         snapshot += 1
203:       end
204:       # Reuse the last key (if there was one)
205:       snapshot -=1 if snapshot > 1 and @plugin_config['overwrite']
206: 
207:       "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}"
208:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 194
194:     def ami_key(appliance_name, path)
195:       base_path = "#{@s3helper.parse_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}"
196: 
197:       return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot']
198: 
199:       @log.info "Determining snapshot name"
200:       snapshot = 1
201:       while @s3helper.stub_s3obj(@bucket, "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/").exists?
202:         snapshot += 1
203:       end
204:       # Reuse the last key (if there was one)
205:       snapshot -=1 if snapshot > 1 and @plugin_config['overwrite']
206: 
207:       "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}"
208:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 136
136:     def asset_bucket(create_if_missing = true, permissions = :private)
137:       @s3helper.bucket(:bucket => @plugin_config['bucket'],
138:         :create_if_missing => create_if_missing,
139:         :acl => permissions,
140:         :location_constraint => @s3_endpoints[@plugin_config['region']][:location]
141:       )
142:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 136
136:     def asset_bucket(create_if_missing = true, permissions = :private)
137:       @s3helper.bucket(:bucket => @plugin_config['bucket'],
138:         :create_if_missing => create_if_missing,
139:         :acl => permissions,
140:         :location_constraint => @s3_endpoints[@plugin_config['region']][:location]
141:       )
142:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 144
144:     def bundle_image(deliverables)
145:       if @plugin_config['snapshot']
146:         @log.debug "Removing bundled image from local disk..."
147:         FileUtils.rm_rf(@ami_build_dir)
148:       end
149: 
150:       return if File.exists?(@ami_build_dir)
151: 
152:       @log.info "Bundling AMI..."
153: 
154:       FileUtils.mkdir_p(@ami_build_dir)
155: 
156:       @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{@s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])
157: 
158:       @log.info "Bundling AMI finished."
159:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 144
144:     def bundle_image(deliverables)
145:       if @plugin_config['snapshot']
146:         @log.debug "Removing bundled image from local disk..."
147:         FileUtils.rm_rf(@ami_build_dir)
148:       end
149: 
150:       return if File.exists?(@ami_build_dir)
151: 
152:       @log.info "Bundling AMI..."
153: 
154:       FileUtils.mkdir_p(@ami_build_dir)
155: 
156:       @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{@s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])
157: 
158:       @log.info "Bundling AMI finished."
159:     end

US constraint is often represented as ’’ or nil

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 211
211:     def constraint_equal?(a, b)
212:       [a, b].collect!{|c| c.nil? ? '': c}
213:       a == b
214:     end

US constraint is often represented as ’’ or nil

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 211
211:     def constraint_equal?(a, b)
212:       [a, b].collect!{|c| c.nil? ? '': c}
213:       a == b
214:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 177
177:     def deregister_image(ami_manifest_key)
178:       if ami = ami_by_manifest_key(ami_manifest_key)
179:         @log.info "Preexisting image '#{ami.location}' for #{@appliance_config.name} will be de-registered, it had id: #{ami.id} (region: #{@plugin_config['region']})."
180:         ami.deregister
181:         @ec2helper.wait_for_image_death(ami)
182:       else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new".
183:         @log.debug "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister"
184:       end
185:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 177
177:     def deregister_image(ami_manifest_key)
178:       if ami = ami_by_manifest_key(ami_manifest_key)
179:         @log.info "Preexisting image '#{ami.location}' for #{@appliance_config.name} will be de-registered, it had id: #{ami.id} (region: #{@plugin_config['region']})."
180:         ami.deregister
181:         @ec2helper.wait_for_image_death(ami)
182:       else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new".
183:         @log.debug "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister"
184:       end
185:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 82
 82:     def execute
 83:       case @type
 84:         when :s3
 85:           upload_to_bucket(@previous_deliverables)
 86:         when :cloudfront
 87:           upload_to_bucket(@previous_deliverables, :public_read)
 88:         when :ami
 89:           ami_dir = ami_key(@appliance_config.name, @plugin_config['path'])
 90:           ami_manifest_key = @s3helper.stub_s3obj(@bucket, "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml")
 91: 
 92:           @log.debug "Going to check whether s3 object exists"
 93: 
 94:           if ami_manifest_key.exists? and @plugin_config['overwrite']
 95:             @log.info "Object exists, attempting to deregister an existing image"
 96:             deregister_image(ami_manifest_key) # Remove existing image
 97:             @s3helper.delete_folder(@bucket, ami_dir) # Avoid triggering dupe detection
 98:           end
 99: 
100:           if !ami_manifest_key.exists? or @plugin_config['snapshot']
101:             @log.info "Doing bundle/snapshot"
102:             bundle_image(@previous_deliverables)
103:             upload_image(ami_dir)
104:           end
105:           register_image(ami_manifest_key)
106:       end
107:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 82
 82:     def execute
 83:       case @type
 84:         when :s3
 85:           upload_to_bucket(@previous_deliverables)
 86:         when :cloudfront
 87:           upload_to_bucket(@previous_deliverables, :public_read)
 88:         when :ami
 89:           ami_dir = ami_key(@appliance_config.name, @plugin_config['path'])
 90:           ami_manifest_key = @s3helper.stub_s3obj(@bucket, "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml")
 91: 
 92:           @log.debug "Going to check whether s3 object exists"
 93: 
 94:           if ami_manifest_key.exists? and @plugin_config['overwrite']
 95:             @log.info "Object exists, attempting to deregister an existing image"
 96:             deregister_image(ami_manifest_key) # Remove existing image
 97:             @s3helper.delete_folder(@bucket, ami_dir) # Avoid triggering dupe detection
 98:           end
 99: 
100:           if !ami_manifest_key.exists? or @plugin_config['snapshot']
101:             @log.info "Doing bundle/snapshot"
102:             bundle_image(@previous_deliverables)
103:             upload_image(ami_dir)
104:           end
105:           register_image(ami_manifest_key)
106:       end
107:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 167
167:     def register_image(ami_manifest_key)
168:       if ami = ami_by_manifest_key(ami_manifest_key)
169:         @log.info "Image for #{@appliance_config.name} is already registered under id: #{ami.id} (region: #{@plugin_config['region']})."
170:       else
171:         ami = @ec2.images.create(:image_location =>  "#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
172:         @ec2helper.wait_for_image_state(:available, ami)
173:         @log.info "Image for #{@appliance_config.name} successfully registered under id: #{ami.id} (region: #{@plugin_config['region']})."
174:       end
175:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 167
167:     def register_image(ami_manifest_key)
168:       if ami = ami_by_manifest_key(ami_manifest_key)
169:         @log.info "Image for #{@appliance_config.name} is already registered under id: #{ami.id} (region: #{@plugin_config['region']})."
170:       else
171:         ami = @ec2.images.create(:image_location =>  "#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
172:         @ec2helper.wait_for_image_state(:available, ami)
173:         @log.info "Image for #{@appliance_config.name} successfully registered under id: #{ami.id} (region: #{@plugin_config['region']})."
174:       end
175:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 161
161:     def upload_image(ami_dir)
162:       @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..."
163: 
164:       @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{@s3_endpoints[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']])
165:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 161
161:     def upload_image(ami_dir)
162:       @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..."
163: 
164:       @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{@s3_endpoints[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']])
165:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 109
109:     def upload_to_bucket(previous_deliverables, permissions = :private)
110:       register_deliverable(
111:           :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz"
112:       )
113: 
114:       # quick and dirty workaround to use @deliverables[:package] later in code
115:       FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package])
116: 
117:       PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package])
118: 
119:       remote_path = "#{@s3helper.parse_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}"
120:       size_m = File.size(@deliverables[:package])/1024**2
121:       s3_obj = @s3helper.stub_s3obj(@bucket,remote_path.gsub(/^\//, '').gsub(/\/\//, ''))
122:       # Does it really exist?
123:       obj_exists = s3_obj.exists?
124: 
125:       if !obj_exists or @plugin_config['overwrite']
126:         @log.info "Will overwrite existing file #{remote_path}" if obj_exists and @plugin_config['overwrite']
127:         @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_m}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..."
128:         s3_obj.write(:file => @deliverables[:package],
129:                         :acl => permissions)
130:         @log.info "Appliance #{@appliance_config.name} uploaded to S3."
131:       else
132:         @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping."
133:       end
134:     end

[Source]

     # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 109
109:     def upload_to_bucket(previous_deliverables, permissions = :private)
110:       register_deliverable(
111:           :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz"
112:       )
113: 
114:       # quick and dirty workaround to use @deliverables[:package] later in code
115:       FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package])
116: 
117:       PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package])
118: 
119:       remote_path = "#{@s3helper.parse_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}"
120:       size_m = File.size(@deliverables[:package])/1024**2
121:       s3_obj = @s3helper.stub_s3obj(@bucket,remote_path.gsub(/^\//, '').gsub(/\/\//, ''))
122:       # Does it really exist?
123:       obj_exists = s3_obj.exists?
124: 
125:       if !obj_exists or @plugin_config['overwrite']
126:         @log.info "Will overwrite existing file #{remote_path}" if obj_exists and @plugin_config['overwrite']
127:         @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_m}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..."
128:         s3_obj.write(:file => @deliverables[:package],
129:                         :acl => permissions)
130:         @log.info "Appliance #{@appliance_config.name} uploaded to S3."
131:       else
132:         @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping."
133:       end
134:     end

[Source]

    # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 39
39:     def validate
40:       set_default_config_value('overwrite', false)
41:       set_default_config_value('path', '/')
42:       set_default_config_value('region', 'us-east-1')
43:       validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
44: 
45:       subtype(:ami) do
46:         set_default_config_value('snapshot', false)
47:         validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
48: 
49:         raise PluginValidationError, "AWS certificate file doesn't exists, please check the path: '#{@plugin_config['cert_file']}'." unless File.exists?(File.expand_path(@plugin_config['cert_file']))
50:         raise PluginValidationError, "AWS key file doesn't exists, please check the path: '#{@plugin_config['key_file']}'." unless File.exists?(File.expand_path(@plugin_config['key_file']))
51:       end
52: 
53:       @s3_endpoints = S3Helper::endpoints
54:       raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin is only aware of the following regions: #{@s3_endpoints.keys.join(", ")}." unless @s3_endpoints.has_key?(@plugin_config['region'])
55: 
56:       @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '')
57: 
58:       # Set global AWS configuration
59:       AWS.config(:access_key_id => @plugin_config['access_key'],
60:         :secret_access_key => @plugin_config['secret_access_key'],
61:         :ec2_endpoint => EC2Helper::endpoints[@plugin_config['region']][:endpoint],
62:         :s3_endpoint => @s3_endpoints[@plugin_config['region']][:endpoint],
63:         :max_retries => 5,
64:         :use_ssl => @plugin_config['use_ssl'])
65: 
66:       @ec2 = AWS::EC2.new
67:       @s3 = AWS::S3.new
68:       @s3helper = S3Helper.new(@ec2, @s3, :log => @log)
69:       @ec2helper = EC2Helper.new(@ec2, :log => @log)
70: 
71:       subtype(:ami) do
72:         # If there is an existing bucket, determine whether its location_constraint matches the region selected
73:         if existing_bucket = asset_bucket(false)
74:           raise PluginValidationError, "Existing bucket #{@plugin_config['bucket']} has a location constraint that does not match the region selected. " <<
75:           "AMI region and bucket location constraint must match." unless constraint_equal?(@s3_endpoints[@plugin_config['region']][:location], existing_bucket.location_constraint)
76:         end
77:       end
78: 
79:       @bucket = asset_bucket(true)
80:     end

[Source]

    # File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 39
39:     def validate
40:       set_default_config_value('overwrite', false)
41:       set_default_config_value('path', '/')
42:       set_default_config_value('region', 'us-east-1')
43:       validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
44: 
45:       subtype(:ami) do
46:         set_default_config_value('snapshot', false)
47:         validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
48: 
49:         raise PluginValidationError, "AWS certificate file doesn't exists, please check the path: '#{@plugin_config['cert_file']}'." unless File.exists?(File.expand_path(@plugin_config['cert_file']))
50:         raise PluginValidationError, "AWS key file doesn't exists, please check the path: '#{@plugin_config['key_file']}'." unless File.exists?(File.expand_path(@plugin_config['key_file']))
51:       end
52: 
53:       @s3_endpoints = S3Helper::endpoints
54:       raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin is only aware of the following regions: #{@s3_endpoints.keys.join(", ")}." unless @s3_endpoints.has_key?(@plugin_config['region'])
55: 
56:       @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '')
57: 
58:       # Set global AWS configuration
59:       AWS.config(:access_key_id => @plugin_config['access_key'],
60:         :secret_access_key => @plugin_config['secret_access_key'],
61:         :ec2_endpoint => EC2Helper::endpoints[@plugin_config['region']][:endpoint],
62:         :s3_endpoint => @s3_endpoints[@plugin_config['region']][:endpoint],
63:         :max_retries => 5,
64:         :use_ssl => @plugin_config['use_ssl'])
65: 
66:       @ec2 = AWS::EC2.new
67:       @s3 = AWS::S3.new
68:       @s3helper = S3Helper.new(@ec2, @s3, :log => @log)
69:       @ec2helper = EC2Helper.new(@ec2, :log => @log)
70: 
71:       subtype(:ami) do
72:         # If there is an existing bucket, determine whether its location_constraint matches the region selected
73:         if existing_bucket = asset_bucket(false)
74:           raise PluginValidationError, "Existing bucket #{@plugin_config['bucket']} has a location constraint that does not match the region selected. " <<
75:           "AMI region and bucket location constraint must match." unless constraint_equal?(@s3_endpoints[@plugin_config['region']][:location], existing_bucket.location_constraint)
76:         end
77:       end
78: 
79:       @bucket = asset_bucket(true)
80:     end

[Validate]