Commit 55f78468 authored by Ted Young's avatar Ted Young Committed by Colin Obyrne

Revert "Revert "refactor UpdateDeployment job to only contain control flow""

And fix the UpdateDeployment job arguments to match new signature and
save off the new cloud_config and manifest text only after successfully
updating the deployment.

This reverts commit 07a0d247.
Signed-off-by: default avatarColin Obyrne <cobyrne@pivotal.io>
parent 50a35c9c
......@@ -294,7 +294,7 @@ module Bosh::Director
deployment = @deployment_manager.find_by_name(params[:deployment_name])
manifest = Psych.load(deployment.manifest)
deployment_plan = DeploymentPlan::Planner.parse(manifest, {}, Config.event_log, Config.logger)
deployment_plan = DeploymentPlan::Planner.parse(manifest, deployment.cloud_config, {}, Config.event_log, Config.logger)
errands = deployment_plan.jobs.select(&:can_run_as_errand?)
......
......@@ -13,13 +13,13 @@ module Bosh::Director
# @param [Hash] manifest Raw deployment manifest
# @return [DeploymentPlan::Planner] Deployment as build from deployment_spec
def parse(manifest, options = {})
def parse(manifest, cloud_config, options = {})
@manifest = manifest
@job_states = safe_property(options, 'job_states',
:class => Hash, :default => {})
@deployment = Planner.new(parse_name, options)
@deployment = Planner.new(parse_name, manifest, cloud_config, options)
parse_properties
parse_releases
......
......@@ -6,6 +6,7 @@ module Bosh::Director
# from the deployment manifest and the running environment.
module DeploymentPlan
class Planner
include LockHelper
include DnsHelper
include ValidationHelper
......@@ -53,16 +54,18 @@ module Bosh::Director
# @param [Logger]
# logger Log for director logging
# @return [Bosh::Director::DeploymentPlan::Planner]
def self.parse(manifest, options, event_log, logger)
def self.parse(manifest, cloud_config, options, event_log, logger)
parser = DeploymentSpecParser.new(event_log, logger)
parser.parse(manifest, options)
parser.parse(manifest, cloud_config, options)
end
def initialize(name, options = {})
def initialize(name, manifest_text, cloud_config, options = {})
raise ArgumentError, 'name must not be nil' unless name
@name = name
@model = nil
@manifest_text = manifest_text
@cloud_config = cloud_config
@model = nil
@properties = {}
@releases = {}
@networks = {}
......@@ -260,6 +263,29 @@ module Bosh::Director
def rename_in_progress?
@job_rename['old_name'] && @job_rename['new_name']
end
def persist_updates!
#prior updates may have had release versions that we no longer use.
#remove the references to these stale releases.
stale_release_versions = (model.release_versions - releases.map(&:model))
stale_release_names = stale_release_versions.map {|version_model| version_model.release.name}
with_release_locks(stale_release_names) do
stale_release_versions.each do |release_version|
model.remove_release_version(release_version)
end
end
model.manifest = Psych.dump(@manifest_text)
model.cloud_config = @cloud_config
model.save
end
def update_stemcell_references!
current_stemcell_models = resource_pools.map { |pool| pool.stemcell.model }
model.stemcells.each do |deployment_stemcell|
deployment_stemcell.remove_deployment(model) unless current_stemcell_models.include?(deployment_stemcell)
end
end
end
end
end
......@@ -11,7 +11,23 @@ module Bosh::Director
@multi_job_updater = multi_job_updater
end
def update
begin
@logger.info('Updating deployment')
assemble
update_jobs
@logger.info('Committing updates')
@deployment_plan.persist_updates!
@logger.info('Finished updating deployment')
ensure
@deployment_plan.update_stemcell_references!
end
end
private
def assemble
@event_log.begin_stage('Preparing DNS', 1)
@base_job.track_and_log('Binding DNS') do
@assembler.bind_dns
......@@ -34,7 +50,9 @@ module Bosh::Director
@base_job.track_and_log('Binding configuration') do
@assembler.bind_configuration
end
end
def update_jobs
@logger.info('Updating jobs')
@multi_job_updater.run(
@base_job,
......
......@@ -24,7 +24,7 @@ module Bosh::Director
def perform
deployment_model = @deployment_manager.find_by_name(@deployment_name)
manifest = Psych.load(deployment_model.manifest)
deployment = DeploymentPlan::Planner.parse(manifest, {}, event_log, logger)
deployment = DeploymentPlan::Planner.parse(manifest, deployment_model.cloud_config, {}, event_log, logger)
job = deployment.job(@errand_name)
if job.nil?
......
......@@ -3,119 +3,96 @@ module Bosh::Director
class UpdateDeployment < BaseJob
include LockHelper
attr_reader :notifier
@queue = :normal
def self.job_type
:update_deployment
end
# @param [String] manifest_file Path to deployment manifest
# @param [Hash] options Deployment options
def initialize(manifest_file, cloud_config_id, options = {})
def initialize(manifest_file_path, cloud_config_id, options = {})
@blobstore = App.instance.blobstores.blobstore
@manifest_file_path = manifest_file_path
@options = options
@cloud_config_id = cloud_config_id
end
logger.info('Reading deployment manifest')
@manifest_file = manifest_file
@manifest = File.read(@manifest_file)
logger.debug("Manifest:\n#{@manifest}")
@cloud_config = Bosh::Director::Models::CloudConfig.find(id: cloud_config_id)
logger.debug("Cloud Config:\n#{@cloud_config.inspect}")
def perform
with_deployment_lock(deployment_plan) do
logger.info('Updating deployment')
notifier.send_start_event
prepare
compile
update
notifier.send_end_event
logger.info('Finished updating deployment')
logger.info('Creating deployment plan')
logger.info("Deployment plan options: #{options.pretty_inspect}")
"/deployments/#{deployment_plan.name}"
end
rescue Exception => e
notifier.send_error_event e
raise e
ensure
FileUtils.rm_rf(@manifest_file_path)
end
plan_options = {
'recreate' => !!options['recreate'],
'job_states' => options['job_states'] || {},
'job_rename' => options['job_rename'] || {}
}
# Job tasks
manifest_as_hash = Psych.load(@manifest)
@deployment_plan = DeploymentPlan::Planner.parse(manifest_as_hash, plan_options, event_log, logger)
logger.info('Created deployment plan')
def prepare
prepare_step = DeploymentPlan::Preparer.new(self, assembler)
prepare_step.prepare
end
nats_rpc = Config.nats_rpc
@notifier = DeploymentPlan::Notifier.new(@deployment_plan, nats_rpc, logger)
def compile
compile_step = PackageCompiler.new(deployment_plan)
compile_step.compile
end
resource_pools = @deployment_plan.resource_pools
@resource_pool_updaters = resource_pools.map do |resource_pool|
def update
resource_pool_updaters = deployment_plan.resource_pools.map do |resource_pool|
ResourcePoolUpdater.new(resource_pool)
end
resource_pools = DeploymentPlan::ResourcePools.new(event_log, resource_pool_updaters)
update_step = DeploymentPlan::Updater.new(self, event_log, resource_pools, assembler, deployment_plan, multi_job_updater)
update_step.update
end
def prepare
@assembler = DeploymentPlan::Assembler.new(@deployment_plan)
preparer = DeploymentPlan::Preparer.new(self, @assembler)
preparer.prepare
# Job dependencies
logger.info('Compiling and binding packages')
PackageCompiler.new(@deployment_plan).compile
def assembler
@assembler ||= DeploymentPlan::Assembler.new(deployment_plan)
end
def update
resource_pools = DeploymentPlan::ResourcePools.new(event_log, @resource_pool_updaters)
job_updater_factory = JobUpdaterFactory.new(@blobstore)
multi_job_updater = DeploymentPlan::BatchMultiJobUpdater.new(job_updater_factory)
updater = DeploymentPlan::Updater.new(self, event_log, resource_pools, @assembler, @deployment_plan, multi_job_updater)
updater.update
def notifier
@notifier ||= DeploymentPlan::Notifier.new(deployment_plan, Config.nats_rpc, logger)
end
def update_stemcell_references
current_stemcells = Set.new
@deployment_plan.resource_pools.each do |resource_pool|
current_stemcells << resource_pool.stemcell.model
end
deployment = @deployment_plan.model
stemcells = deployment.stemcells
stemcells.each do |stemcell|
unless current_stemcells.include?(stemcell)
stemcell.remove_deployment(deployment)
end
def deployment_plan
@deployment_plan ||= begin
logger.info('Reading deployment manifest')
manifest_text = File.read(@manifest_file_path)
logger.debug("Manifest:\n#{manifest_text}")
deployment_manifest = Psych.load(manifest_text)
plan_options = {
'recreate' => !!@options['recreate'],
'job_states' => @options['job_states'] || {},
'job_rename' => @options['job_rename'] || {}
}
logger.info('Creating deployment plan')
logger.info("Deployment plan options: #{plan_options.pretty_inspect}")
cloud_config = Bosh::Director::Models::CloudConfig[@cloud_config_id]
plan = DeploymentPlan::Planner.parse(deployment_manifest, cloud_config, plan_options, event_log, logger)
logger.info('Created deployment plan')
plan
end
end
def perform
with_deployment_lock(@deployment_plan) do
logger.info('Preparing deployment')
notifier.send_start_event
prepare
begin
deployment = @deployment_plan.model
logger.info('Finished preparing deployment')
logger.info('Updating deployment')
update
with_release_locks(@deployment_plan.releases.map(&:name)) do
deployment.db.transaction do
deployment.remove_all_release_versions
# Now we know that deployment has succeeded and can remove
# previous partial deployments release version references
# to be able to delete these release versions later.
@deployment_plan.releases.each do |release|
deployment.add_release_version(release.model)
end
end
end
deployment.manifest = @manifest
deployment.cloud_config = @cloud_config
deployment.save
notifier.send_end_event
logger.info('Finished updating deployment')
"/deployments/#{deployment.name}"
ensure
update_stemcell_references
end
def multi_job_updater
@multi_job_updater ||= begin
DeploymentPlan::BatchMultiJobUpdater.new(JobUpdaterFactory.new(@blobstore))
end
rescue Exception => e
notifier.send_error_event e
raise e
ensure
FileUtils.rm_rf(@manifest_file)
end
end
end
......
......@@ -202,4 +202,74 @@ def check_event_log
yield events
end
def strip_heredoc(str)
indent = str.scan(/^[ \t]*(?=\S)/).min.size || 0
str.gsub(/^[ \t]{#{indent}}/, '')
end
module ManifestHelper
class << self
def default_deployment_manifest(overrides = {})
{
'name' => 'deployment-name',
'releases' => [release],
'update' => {
'max_in_flight' => 10,
'canaries' => 2,
'canary_watch_time' => 1000,
'update_watch_time' => 1000,
},
}.merge(overrides)
end
def default_legacy_manifest(overrides = {})
(default_deployment_manifest.merge(default_iaas_manifest)).merge(overrides)
end
def default_iaas_manifest(overrides = {})
{
'networks' => [ManifestHelper::network],
'resource_pools' => [ManifestHelper::resource_pool],
'compilation' => {
'workers' => 1,
'network'=>'network-name',
'cloud_properties' => {},
},
}.merge(overrides)
end
def release(overrides = {})
{
'name' => 'release-name',
'version' => 'latest',
}.merge(overrides)
end
def network(overrides = {})
{ 'name' => 'network-name', 'subnets' => [] }.merge(overrides)
end
def disk_pool(name='dp-name')
{'name' => name, 'disk_size' => 10000}.merge(overrides)
end
def job(overrides = {})
{
'name' => 'job-name',
'resource_pool' => 'rp-name',
'instances' => 1,
'networks' => [{'name' => 'network-name'}],
'templates' => [{'name' => 'template-name'}]
}.merge(overrides)
end
def resource_pool(overrides = {})
{
'name' => 'rp-name',
'network'=>'network-name',
'stemcell'=> {'name' => 'default','version'=>'1'},
'cloud_properties'=>{}
}.merge(overrides)
end
end
end
require 'spec_helper'
module Support
module FileHelpers
class DeploymentDirectory
attr_reader :path, :artifacts_dir, :tarballs
def initialize
@path = Dir.mktmpdir('deployment-path')
end
def add_file(filepath, contents = nil)
full_path = File.join(path, filepath)
FileUtils.mkdir_p(File.dirname(full_path))
if contents
File.open(full_path, 'w') { |f| f.write(contents) }
else
FileUtils.touch(full_path)
end
full_path
end
end
end
end
RSpec.configure do |config|
config.include(Support::FileHelpers)
end
......@@ -419,8 +419,10 @@ module Bosh::Director
Models::Deployment.make(
name: 'fake-dep-name',
manifest: "---\nmanifest: true",
cloud_config: cloud_config
)
end
let(:cloud_config) { Models::CloudConfig.create }
before { allow(Config).to receive(:event_log).with(no_args).and_return(event_log) }
let(:event_log) { instance_double('Bosh::Director::EventLog::Log') }
......@@ -429,7 +431,7 @@ module Bosh::Director
before do
allow(DeploymentPlan::Planner).to receive(:parse).
with({'manifest' => true}, {}, event_log, logger).
with({'manifest' => true}, cloud_config, {}, event_log, logger).
and_return(deployment)
end
let(:deployment) { instance_double('Bosh::Director::DeploymentPlan::Planner', name: 'deployment') }
......
......@@ -180,7 +180,8 @@ module Bosh::Director
end
describe '#bind_idle_vm' do
let(:deployment_plan) { Planner.new('fake-deployment') }
let(:cloud_config){ Models::CloudConfig.create }
let(:deployment_plan) { Planner.new('fake-deployment', '{"fake-manifest-text": true}', cloud_config) }
let(:state) { { 'state' => 'foo' } }
let(:network) { Network.new(deployment_plan, {'name' => resource_pool_manifest['network']}) }
let(:resource_pool) { ResourcePool.new(deployment_plan, resource_pool_manifest, logger) }
......
......@@ -32,17 +32,18 @@ module Bosh::Director
canonical_name: 'fake-network-name',
})
end
let(:cloud_config) { Models::CloudConfig.create }
describe 'name key' do
it 'parses name' do
deployment_spec.merge!('name' => 'Name with spaces')
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.name).to eq('Name with spaces')
end
it 'sets canonical name' do
deployment_spec.merge!('name' => 'Name with spaces')
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.canonical_name).to eq('namewithspaces')
end
end
......@@ -50,13 +51,13 @@ module Bosh::Director
describe 'properties key' do
it 'parses basic properties' do
deployment_spec.merge!('properties' => { 'foo' => 'bar' })
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.properties).to eq('foo' => 'bar')
end
it 'allows to not include properties key' do
deployment_spec.delete('properties')
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.properties).to eq({})
end
end
......@@ -80,7 +81,7 @@ module Bosh::Director
with(be_a(Planner), 'name' => 'rv-name').
and_return(rv)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.releases).to eq([rv])
end
......@@ -89,7 +90,7 @@ module Bosh::Director
with(be_a(Planner), 'name' => 'rv-name').
and_return(rv)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.release('rv-name')).to eq(rv)
end
end
......@@ -117,7 +118,7 @@ module Bosh::Director
with(be_a(Planner), 'name' => 'rv2-name').
and_return(rv2)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.releases).to eq([rv1, rv2])
end
......@@ -130,7 +131,7 @@ module Bosh::Director
with(be_a(Planner), 'name' => 'rv2-name').
and_return(rv2)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.release('rv1-name')).to eq(rv1)
expect(deployment.release('rv2-name')).to eq(rv2)
end
......@@ -152,7 +153,7 @@ module Bosh::Director
and_return(rv)
expect {
parser.parse(deployment_spec)
parser.parse(deployment_spec, cloud_config)
}.to raise_error(/duplicate release name/i)
end
end
......@@ -164,7 +165,7 @@ module Bosh::Director
it 'raises an error' do
expect {
parser.parse(deployment_spec)
parser.parse(deployment_spec, cloud_config)
}.to raise_error(/use one of the two/)
end
end
......@@ -175,7 +176,7 @@ module Bosh::Director
it 'raises an error' do
expect {
parser.parse(deployment_spec)
parser.parse(deployment_spec, cloud_config)
}.to raise_error(
ValidationMissingField,
/Required property `releases' was not specified in object .+/,
......@@ -195,7 +196,7 @@ module Bosh::Director
with(be_a(Planner), 'foo' => 'bar').
and_return(compilation)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.compilation).to eq(compilation)
end
end
......@@ -205,7 +206,7 @@ module Bosh::Director
it 'raises an error' do
expect {
parser.parse(deployment_spec)
parser.parse(deployment_spec, cloud_config)
}.to raise_error(
ValidationMissingField,
/Required property `compilation' was not specified in object .+/,
......@@ -225,7 +226,7 @@ module Bosh::Director
with('foo' => 'bar').
and_return(update)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.update).to eq(update)
end
end
......@@ -235,7 +236,7 @@ module Bosh::Director
it 'raises an error' do
expect {
parser.parse(deployment_spec)
parser.parse(deployment_spec, cloud_config)
}.to raise_error(
ValidationMissingField,
/Required property `update' was not specified in object .+/,
......@@ -261,13 +262,13 @@ module Bosh::Director
with(be_a(Planner), 'foo' => 'bar').
and_return(network)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.networks).to eq([network])
end
it 'allows to look up network by name' do
allow(ManualNetwork).to receive(:new).and_return(network)
deployment = parser.parse(deployment_spec)
deployment = parser.parse(deployment_spec, cloud_config)
expect(deployment.network('fake-net-name')).to eq(network)
end
end
......@@ -294,7 +295,7 @@ module Bosh::Director
end
expect {
parser.parse(deployment_spec)
parser.parse(deployment_spec, cloud_config)
}.to raise_error(
DeploymentCanonicalNetworkNameTaken,
"Invalid network name `Bar', canonical name already taken",
......@@ -308,7 +309,7 @@ module Bosh::Director