seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Finished anvil-migrate-server and anvil-safe-start! Lots of testing still needed for both though, and 'anvil-safe-start' does run as a job yet, but the logic is all there.

Browse Source 
* Fixed a bug in Cluster->migrate_server() where waiting for the server to migate would never exit.

Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 4 years ago
parent a6a11abe01
commit 711a04999e
9 changed files with 380 additions and 141 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      Anvil/Tools.pm
  2. 10
      Anvil/Tools/Cluster.pm
  3. 2
      ocf/alteeve/server
  4. 18
      share/words.xml
  5. 11
      tools/anvil-boot-server
  6. 435
      tools/anvil-migrate-server
  7. 2
      tools/anvil-rename-server
  8. 25
      tools/anvil-safe-start
  9. 15
      tools/anvil-shutdown-server

3
Anvil/Tools.pm
Unescape View File

@ -1089,6 +1089,7 @@ sub _set_paths
}, },
exe => { exe => {
ocf_alteeve => "/usr/lib/ocf/resource.d/alteeve/server", ocf_alteeve => "/usr/lib/ocf/resource.d/alteeve/server",
'anvil-boot-server' => "/usr/sbin/anvil-boot-server",
'anvil-change-password' => "/usr/sbin/anvil-change-password", 'anvil-change-password' => "/usr/sbin/anvil-change-password",
'anvil-check-memory' => "/usr/sbin/anvil-check-memory", 'anvil-check-memory' => "/usr/sbin/anvil-check-memory",
'anvil-configure-host' => "/usr/sbin/anvil-configure-host", 'anvil-configure-host' => "/usr/sbin/anvil-configure-host",
@ -1101,9 +1102,11 @@ sub _set_paths
'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall", 'anvil-manage-firewall' => "/usr/sbin/anvil-manage-firewall",
'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys", 'anvil-manage-keys' => "/usr/sbin/anvil-manage-keys",
'anvil-manage-power' => "/usr/sbin/anvil-manage-power", 'anvil-manage-power' => "/usr/sbin/anvil-manage-power",
'anvil-migrate-server' => "/usr/sbin/anvil-migrate-server",
'anvil-parse-fence-agents' => "/usr/sbin/anvil-parse-fence-agents", 'anvil-parse-fence-agents' => "/usr/sbin/anvil-parse-fence-agents",
'anvil-provision-server' => "/usr/sbin/anvil-provision-server", 'anvil-provision-server' => "/usr/sbin/anvil-provision-server",
'anvil-report-memory' => "/usr/sbin/anvil-report-memory", 'anvil-report-memory' => "/usr/sbin/anvil-report-memory",
'anvil-shutdown-server' => "/usr/sbin/anvil-shutdown-server",
'anvil-sync-shared' => "/usr/sbin/anvil-sync-shared", 'anvil-sync-shared' => "/usr/sbin/anvil-sync-shared",
'anvil-update-files' => "/usr/sbin/anvil-update-files", 'anvil-update-files' => "/usr/sbin/anvil-update-files",
'anvil-update-states' => "/usr/sbin/anvil-update-states", 'anvil-update-states' => "/usr/sbin/anvil-update-states",

10
Anvil/Tools/Cluster.pm
Unescape View File

@ -1614,18 +1614,18 @@ sub migrate_server
while($waiting) while($waiting)
{ {
$anvil->Cluster->parse_cib({debug => $debug}); $anvil->Cluster->parse_cib({debug => $debug});
my $status = $anvil->data->{cib}{parsed}{data}{server}{$server}{status}; my $status = $anvil->data->{cib}{parsed}{data}{server}{$server}{status};
my $host = $anvil->data->{cib}{parsed}{data}{server}{$server}{host_name}; my $host_name = $anvil->data->{cib}{parsed}{data}{server}{$server}{host_name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => $debug, list => {
status => $status, status => $status,
host => $host, host_name => $host_name,
}}); }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0550", variables => { $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0550", variables => {
server => $server, server => $server,
requested_node => $node, requested_node => $node,
}}); }});
if (($host eq "running") && ($host eq $node)) if (($status eq "running") && ($host_name eq $node))
{ {
# It's done. # It's done.
$waiting = 0; $waiting = 0;

2
ocf/alteeve/server
Unescape View File

@ -995,7 +995,7 @@ sub stop_server
} }
# Now stop the DRBD resource(s). # Now stop the DRBD resource(s).
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 3, list => {
'environment::OCF_RESKEY_CRM_meta_stop_drbd_resources' => $anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources}, 'environment::OCF_RESKEY_CRM_meta_stop_drbd_resources' => $anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources},
}}); }});
if ($anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources}) if ($anvil->data->{environment}{OCF_RESKEY_CRM_meta_stop_drbd_resources})

18
share/words.xml
Unescape View File

@ -375,6 +375,18 @@ The attempt to start the cluster appears to have failed. The return code '0' was
<key name="error_0266">The server: [#!variable!server!#] does not appear to be in the cluster. Unable to shut it down.</key> <key name="error_0266">The server: [#!variable!server!#] does not appear to be in the cluster. Unable to shut it down.</key>
<key name="error_0267">The server: [#!variable!server!#] failed to boot. The reason why should be in the logs.</key> <key name="error_0267">The server: [#!variable!server!#] failed to boot. The reason why should be in the logs.</key>
<key name="error_0268">The server: [#!variable!server!#] failed to shut down. The reason why should be in the logs.</key> <key name="error_0268">The server: [#!variable!server!#] failed to shut down. The reason why should be in the logs.</key>
<key name="error_0269">The server UUID: [#!variable!server_uuid!#] is not valid.</key>
<key name="error_0270"><![CDATA[No server specified to migrate. Please use '--server <name|all>' or '--server-uuid <UUID>.]]></key>
<key name="error_0271">This host is not a node, unable to migrate servers.</key>
<key name="error_0272"><![CDATA[No target node specified. Please use '--target <node_name>'.]]></key>
<key name="error_0273">The target: [#!variable!target!#] appears to be invalid. The --target switch needs to be set to 'peer', 'local', '#!variable!local_name!#' or '#!variable!peer_name!#'.</key>
<key name="error_0274">The server: [#!variable!server!#] failed to migrate. The reason why should be in the logs.</key>
<key name="error_0275">
The attempt to start the servers appears to have failed. The return code '0' was expected, but: [#!variable!return_code!#] was received. The output was:
====
#!variable!output!#
====
</key>
<!-- Files templates --> <!-- Files templates -->
<!-- NOTE: Translating these files requires an understanding of which likes are translatable --> <!-- NOTE: Translating these files requires an understanding of which likes are translatable -->
@ -782,6 +794,11 @@ It should be provisioned in the next minute or two.</key>
<key name="job_0287">The server: [#!variable!server!#] has been asked to boot. It should come up soon.</key> <key name="job_0287">The server: [#!variable!server!#] has been asked to boot. It should come up soon.</key>
<key name="job_0288">The server: [#!variable!server!#] will now be booted...</key> <key name="job_0288">The server: [#!variable!server!#] will now be booted...</key>
<key name="job_0289">The server: [#!variable!server!#] will now be asked to shut down. If the server doesn't stop, please log into it and make sure it reacted to the power button event. Shut it down manually, if needed.</key> <key name="job_0289">The server: [#!variable!server!#] will now be asked to shut down. If the server doesn't stop, please log into it and make sure it reacted to the power button event. Shut it down manually, if needed.</key>
<key name="job_0290">Booting server(s)...</key>
<key name="job_0291">Source node: [#!variable!source!#], target node is: [#!variable!target!#].</key>
<key name="job_0292">The server: [#!variable!server!#] has been migrated to: [#!variable!target!#].</key>
<key name="job_0293">The server: [#!variable!server!#] will now be migrated to: [#!variable!target!#]. This could take some time! How much RAM is allocated to this server, the speed of the back-channel network and how busy the server is all contribute to migration time. Please be patient!</key>
<key name="job_0294">The server: [#!variable!server!#] has been asked to migrate. We are not waiting for it to complete.</key>
<!-- Log entries --> <!-- Log entries -->
<key name="log_0001">Starting: [#!variable!program!#].</key> <key name="log_0001">Starting: [#!variable!program!#].</key>
@ -1489,6 +1506,7 @@ The file: [#!variable!file!#] needs to be updated. The difference is:
<key name="log_0611">We're online as: [#!variable!node_name!#] and quorate!</key> <key name="log_0611">We're online as: [#!variable!node_name!#] and quorate!</key>
<key name="log_0612">We're not online yet. Waiting for 'in_ccm/crmd/join': [#!variable!in_ccm!#/#!variable!crmd!#/#!variable!join!#]. ('in_ccm' = consensus cluster member, communication layer. 'crmd' = cluster resource manager daemon is up, 'join' = allowed to host resources).</key> <key name="log_0612">We're not online yet. Waiting for 'in_ccm/crmd/join': [#!variable!in_ccm!#/#!variable!crmd!#/#!variable!join!#]. ('in_ccm' = consensus cluster member, communication layer. 'crmd' = cluster resource manager daemon is up, 'join' = allowed to host resources).</key>
<key name="log_0613">The file: [#!variable!file_name!#] is not recorded for the Anvil! [#!variable!anvil_name!#] yet. Registering it now as not sync'ed to this Anvil! system.</key> <key name="log_0613">The file: [#!variable!file_name!#] is not recorded for the Anvil! [#!variable!anvil_name!#] yet. Registering it now as not sync'ed to this Anvil! system.</key>
<key name="log_0614">Asking 'anvil-boot-server' to boot the servers now.</key>
<!-- Messages for users (less technical than log entries), though sometimes used for logs, too. --> <!-- Messages for users (less technical than log entries), though sometimes used for logs, too. -->
<key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key> <key name="message_0001">The host name: [#!variable!target!#] does not resolve to an IP address.</key>

11
tools/anvil-boot-server
Unescape View File

@ -48,7 +48,7 @@ if (not $anvil->data->{sys}{database}{connections})
{ {
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try
# again after we exit. # again after we exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0077"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"});
sleep 10; sleep 10;
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
@ -111,6 +111,15 @@ if ($anvil->data->{switches}{'server-uuid'})
'switches::server' => $anvil->data->{switches}{'server'}, 'switches::server' => $anvil->data->{switches}{'server'},
}}); }});
} }
else
{
# Invalid server UUID.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0269", variables => {
server_uuid => $anvil->data->{switches}{'server-uuid'},
}});
$anvil->Job->update_progress({progress => 100, message => "error_0269,!!server_uuid!".$anvil->data->{switches}{'server-uuid'}."!!"});
$anvil->nice_exit({exit_code => 1});
}
} }
# Do we have a server name? # Do we have a server name?

435
tools/anvil-migrate-server
Unescape View File

@ -28,8 +28,20 @@ my $anvil = Anvil::Tools->new();
# Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is # Read switches (target ([user@]host[:port]) and the file with the target's password. If the password is
# passed directly, it will be used. Otherwise, the password will be read from the database. # passed directly, it will be used. Otherwise, the password will be read from the database.
$anvil->data->{switches}{'job-uuid'} = ""; $anvil->data->{switches}{'job-uuid'} = "";
$anvil->data->{switches}{'no-wait'} = ""; # We normall wait for each migation to finish. This skips that. With '--all', this causes all migrations to run in parallel
$anvil->data->{switches}{'parallel'} = "";
$anvil->data->{switches}{'server'} = "";
$anvil->data->{switches}{'server-uuid'} = "";
$anvil->data->{switches}{'target'} = "";
$anvil->Get->switches; $anvil->Get->switches;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'},
'switches::no-wait' => $anvil->data->{switches}{'no-wait'},
'switches::server' => $anvil->data->{switches}{'server'},
'switches::server-uuid' => $anvil->data->{switches}{'server-uuid'},
'switches::target' => $anvil->data->{switches}{'target'},
}});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, secure => 0, key => "log_0115", variables => { program => $THIS_FILE }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'switches::job-uuid' => $anvil->data->{switches}{'job-uuid'} }});
@ -37,15 +49,13 @@ $anvil->Database->connect();
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"});
if (not $anvil->data->{sys}{database}{connections}) if (not $anvil->data->{sys}{database}{connections})
{ {
# No databases, sleep for a bit and then exit. The daemon will retry after we exit. # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0218"}); # again after we exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"});
sleep 10; sleep 10;
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
$anvil->data->{sys}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'sys::anvil_uuid' => $anvil->data->{sys}{anvil_uuid} }});
# If we don't have a job UUID, try to find one. # If we don't have a job UUID, try to find one.
if (not $anvil->data->{switches}{'job-uuid'}) if (not $anvil->data->{switches}{'job-uuid'})
{ {
@ -54,7 +64,6 @@ if (not $anvil->data->{switches}{'job-uuid'})
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::job-uuid" => $anvil->data->{switches}{'job-uuid'} }});
} }
# If we still don't have a job-uuit, go into interactive mode.
if ($anvil->data->{switches}{'job-uuid'}) if ($anvil->data->{switches}{'job-uuid'})
{ {
# Load the job data. # Load the job data.
@ -64,44 +73,114 @@ if ($anvil->data->{switches}{'job-uuid'})
progress => 1, progress => 1,
job_picked_up_by => $$, job_picked_up_by => $$,
job_picked_up_at => time, job_picked_up_at => time,
message => "message_0218", message => "job_0290",
}); });
# Are we in an Anvil! system? # Pull out the job data.
if (not $anvil->data->{sys}{anvil_uuid}) foreach my $line (split/\n/, $anvil->data->{jobs}{job_data})
{ {
# We're not in an Anvil!. if ($line =~ /server=(.*?)$/)
if ($anvil->data->{switches}{'job-uuid'})
{ {
# Mark the job as failed. $anvil->data->{switches}{'server'} = $1;
$anvil->Job->update_progress({ $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
progress => 100, 'switches::server' => $anvil->data->{switches}{'server'},
message => "error_0231", }});
job_status => "failed", }
}); if ($line =~ /server-uuid=(.*?)$/)
{
$anvil->data->{switches}{'server-uuid'} = $1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'switches::server-uuid' => $anvil->data->{switches}{'server-uuid'},
}});
} }
# Log an exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0231"});
$anvil->nice_exit({exit_code => 1});
} }
# Job data will be in $anvil->data->{jobs}{job_data}
run_jobs($anvil);
} }
else
# Now check that we have a server. If it's a server_uuid, read the server name.
if ($anvil->data->{switches}{'server-uuid'})
{ {
if (not $anvil->data->{sys}{anvil_uuid}) # Convert the server_uuid to a server_name.
my $query = "SELECT server_name FROM servers WHERE server_uuid = ".$anvil->Database->quote($anvil->data->{switches}{'server-uuid'}).";";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { query => $query }});
my $server_name = $anvil->Database->query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0];
$server_name = "" if not defined $server_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_name => $server_name }});
if ($server_name)
{ {
# We can't do anything, exit. $anvil->data->{switches}{'server'} = $server_name;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => 'err', key => "error_0231"}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
'switches::server' => $anvil->data->{switches}{'server'},
}});
}
else
{
# Invalid server UUID.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0269", variables => {
server_uuid => $anvil->data->{switches}{'server-uuid'},
}});
$anvil->Job->update_progress({progress => 100, message => "error_0269,!!server_uuid!".$anvil->data->{switches}{'server-uuid'}."!!"});
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
# Interactive!
ask_for_server($anvil);
} }
# Do we have a server name?
if (not $anvil->data->{switches}{'server'})
{
# Unable to proceed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0270"});
$anvil->Job->update_progress({progress => 100, message => "error_0270"});
$anvil->nice_exit({exit_code => 1});
}
# Are we a node or DR host?
$anvil->data->{sys}{host_type} = $anvil->Get->host_type();
if ($anvil->data->{sys}{host_type} ne "node")
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0271"});
$anvil->Job->update_progress({progress => 100, message => "error_0271"});
$anvil->nice_exit({exit_code => 1});
}
# Make sure that we're in an Anvil! system.
$anvil->data->{sys}{anvil_uuid} = $anvil->Cluster->get_anvil_uuid();
if (not $anvil->data->{sys}{anvil_uuid})
{
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0260"});
$anvil->Job->update_progress({progress => 100, message => "error_0260"});
$anvil->nice_exit({exit_code => 1});
}
# Do we have a target?
if (not $anvil->data->{switches}{'target'})
{
# Unable to proceed.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0272"});
$anvil->Job->update_progress({progress => 100, message => "error_0272"});
$anvil->nice_exit({exit_code => 1});
}
# This is copied from anvil-boot-server, but it works here as well. We can't use 'pcs' without pacemaker
# being up.
wait_for_pacemaker($anvil);
# Make sure the target node is valid.
find_target($anvil);
# If 'server' is 'all', migrate all servers.
if (lc($anvil->data->{switches}{'server'}) eq "all")
{
migrate_all_servers($anvil);
}
else
{
migrate_server($anvil, $anvil->data->{switches}{'server'}, 50);
}
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0281"});
$anvil->Job->update_progress({progress => 100, message => "job_0281"});
$anvil->nice_exit({exit_code => 0}); $anvil->nice_exit({exit_code => 0});
@ -109,132 +188,236 @@ $anvil->nice_exit({exit_code => 0});
# Functions # # Functions #
############################################################################################################# #############################################################################################################
sub migrate_server
# This actually provisions a VM.
sub run_jobs
{ {
my ($anvil) = @_; my ($anvil, $server, $progress) = @_;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
server => $server,
progress => $progress,
}});
# This parses the jobs::job_data intp variables. # Is the server in the cluster?
parse_job_data($anvil); if (not exists $anvil->data->{cib}{parsed}{data}{server}{$server})
{
# Nope.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "log_0548", variables => { server => $server }});
$anvil->Job->update_progress({progress => 100, message => "log_0548,!!server!".$server."!!"});
$anvil->nice_exit({exit_code => 1});
}
my $status = $anvil->data->{cib}{parsed}{data}{server}{$server}{status};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { status => $status }});
if ($status eq "off")
{
# It's off already
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0284", variables => { server => $server }});
$anvil->Job->update_progress({progress => $progress, message => "job_0284,!!server!".$server."!!"});
return(0);
}
### TODO: Record past migration times so that we give the user a estimate of how long it will take.
# Now migrate.
my $wait = $anvil->data->{switches}{'no-wait'} ? 0 : 1;
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0293", variables => {
server => $server,
target => $anvil->data->{sys}{target},
}});
$anvil->Job->update_progress({progress => $progress, message => "job_0293,!!server!".$server."!!,!!target!".$anvil->data->{sys}{target}."!!"});
my $problem = $anvil->Cluster->migrate_server({
debug => 2,
server => $server,
node => $anvil->data->{sys}{target},
'wait' => $wait,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if ($problem)
{
# Failed, abort.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0274", variables => { server => $server }});
$anvil->Job->update_progress({progress => 100, message => "error_0274,!!server!".$server."!!"});
$anvil->nice_exit({exit_code => 1});
}
else
{
if ($wait)
{
# Migrated!
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0292", variables => {
server => $server,
target => $anvil->data->{sys}{target},
}});
$anvil->Job->update_progress({progress => $progress, message => "job_0292,!!server!".$server."!!,!!target!".$anvil->data->{sys}{target}."!!"});
}
else
{
# Migration requested.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0294", variables => { server => $server }});
$anvil->Job->update_progress({progress => $progress, message => "job_0294,!!server!".$server."!!"});
}
}
return(0); return(0);
} }
# This parses and verifies the job data sub migrate_all_servers
sub parse_job_data
{ {
my ($anvil) = @_; my ($anvil) = @_;
$anvil->data->{job}{server_uuid} = ""; # We top out at 90, bottom is 20.
$anvil->data->{job}{peer_mode} = 0; my $server_count = keys %{$anvil->data->{cib}{parsed}{data}{server}};
foreach my $line (split/\n/, $anvil->data->{jobs}{job_data}) my $increment = int(70 / $server_count);
my $percent = 15;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
server_count => $server_count,
increment => $increment,
}});
foreach my $server (sort {$a cmp $b} keys %{$anvil->data->{cib}{parsed}{data}{server}})
{ {
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); my $status = $anvil->data->{cib}{parsed}{data}{server}{$server}{status};
if ($line =~ /server_uuid=(.*)$/) my $host_name = $anvil->data->{cib}{parsed}{data}{server}{$server}{host_name};
{ my $role = $anvil->data->{cib}{parsed}{data}{server}{$server}{role};
$anvil->data->{job}{server_uuid} = $1; my $active = $anvil->data->{cib}{parsed}{data}{server}{$server}{active};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::server_uuid' => $anvil->data->{job}{server_uuid} }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
} 's1:server' => $server,
if ($line =~ /target_host_uuid=(.*)$/) 's2:status' => $status,
's2:host_name' => $host_name,
's4:role' => $role,
's5:active' => $active,
}});
if ($status ne "off")
{ {
$anvil->data->{job}{target_host_uuid} = 1; # Migrate it.
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'job::target_host_uuid' => $anvil->data->{job}{target_host_uuid} }}); $percent += $increment;
migrate_server($anvil, $server, $percent);
} }
} }
# Did we get a server UUID? return(0);
if (not $anvil->data->{job}{server_uuid}) }
sub find_target
{
my ($anvil) = @_;
# Convert the host names to short names for easier matching.
my $short_target_name = $anvil->data->{switches}{'target'};
$short_target_name =~ s/\..*$//;
my $short_local_name = $anvil->data->{cib}{parsed}{'local'}{name};
$short_local_name =~ s/\..*$//;
my $short_peer_name = $anvil->data->{cib}{parsed}{peer}{name};
$short_peer_name =~ s/\..*$//;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:short_target_name" => $short_target_name,
"s2:short_local_name" => $short_local_name,
"s3:short_peer_name" => $short_peer_name,
}});
$anvil->data->{sys}{source} = "";
$anvil->data->{sys}{target} = "";
if (lc($anvil->data->{switches}{'target'}) eq "peer")
{ {
$anvil->Job->update_progress({ # Migrate to the peer.
progress => 100, $anvil->data->{sys}{source} = $anvil->data->{cib}{parsed}{'local'}{name};
message => "error_0232,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", $anvil->data->{sys}{target} = $anvil->data->{cib}{parsed}{peer}{name};
job_status => "failed", $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
}); "sys::source" => $anvil->data->{sys}{source},
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0232", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }}); "sys::target" => $anvil->data->{sys}{target},
$anvil->nice_exit({exit_code => 1}); }});
} }
elsif ((lc($anvil->data->{switches}{'target'}) eq "local") or (lc($anvil->data->{switches}{'target'}) eq "here"))
# Does the server UUID match to a server?
$anvil->Database->get_servers();
my $server_uuid = $anvil->data->{job}{server_uuid};
if (($server_uuid ne "all") && (not exists $anvil->data->{servers}{server_uuid}{$server_uuid}))
{ {
# Server UUID is invalid # Migrate to the this machine.
$anvil->Job->update_progress({ $anvil->data->{sys}{source} = $anvil->data->{cib}{parsed}{peer}{name};
progress => 100, $anvil->data->{sys}{target} = $anvil->data->{cib}{parsed}{'local'}{name};
message => "error_0220,!!server_uuid!".$server_uuid."!!", $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
job_status => "failed", "sys::source" => $anvil->data->{sys}{source},
}); "sys::target" => $anvil->data->{sys}{target},
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0220", variables => { server_uuid => $server_uuid }}); }});
$anvil->nice_exit({exit_code => 1});
} }
elsif (lc($short_target_name) eq lc($short_peer_name))
# Is the target_host_uuid a host in our Anvil! system?
if (not $anvil->data->{job}{target_host_uuid})
{ {
$anvil->Job->update_progress({ # Migrate to the peer.
progress => 100, $anvil->data->{sys}{source} = $anvil->data->{cib}{parsed}{'local'}{name};
message => "error_0234,!!job_uuid!".$anvil->data->{switches}{'job-uuid'}."!!", $anvil->data->{sys}{target} = $anvil->data->{cib}{parsed}{peer}{name};
job_status => "failed", $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
}); "sys::source" => $anvil->data->{sys}{source},
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0234", variables => { job_uuid => $anvil->data->{switches}{'job-uuid'} }}); "sys::target" => $anvil->data->{sys}{target},
$anvil->nice_exit({exit_code => 1}); }});
} }
else elsif (lc($short_target_name) eq lc($short_local_name))
{ {
my $anvil_uuid = $anvil->data->{sys}{anvil_uuid}; # Migrate to the this machine.
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; $anvil->data->{sys}{source} = $anvil->data->{cib}{parsed}{peer}{name};
my $node2_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node2_host_uuid}; $anvil->data->{sys}{target} = $anvil->data->{cib}{parsed}{'local'}{name};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
anvil_uuid => $anvil_uuid, "sys::source" => $anvil->data->{sys}{source},
node1_host_uuid => $node1_host_uuid, "sys::target" => $anvil->data->{sys}{target},
node2_host_uuid => $node2_host_uuid,
}}); }});
if (($anvil->data->{job}{target_host_uuid} ne $node1_host_uuid) &&
($anvil->data->{job}{target_host_uuid} ne $node2_host_uuid))
{
$anvil->Job->update_progress({
progress => 100,
message => "error_0235,!!target_host_uuid!".$anvil->data->{job}{target_host_uuid}."!!",
job_status => "failed",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0235", variables => { target_host_uuid => $anvil->data->{job}{target_host_uuid} }});
$anvil->nice_exit({exit_code => 1});
}
} }
else
{
# No match.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0273", variables => {
target => $anvil->data->{switches}{'target'},
local_name => $anvil->data->{cib}{parsed}{'local'}{name},
peer_name => $anvil->data->{cib}{parsed}{peer}{name},
}});
$anvil->Job->update_progress({progress => 100, message => "error_0273,!!target!".$anvil->data->{switches}{'target'}."!!,!!local_name!".$anvil->data->{cib}{parsed}{'local'}{name}."!!,!!peer_name!".$anvil->data->{cib}{parsed}{peer}{name}."!!"});
$anvil->nice_exit({exit_code => 1});
}
# Record that we have our target.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0291", variables => {
source => $anvil->data->{sys}{source},
target => $anvil->data->{sys}{target},
}});
$anvil->Job->update_progress({progress => 18, message => "job_0291,!!source!".$anvil->data->{sys}{source}."!!,!!target!".$anvil->data->{sys}{target}."!!"});
my $host_type = $anvil->Get->host_type(); return(0);
if ($host_type eq "node") }
sub wait_for_pacemaker
{
my ($anvil) = @_;
# Wait for the node to be up.
my $waiting = 1;
while($waiting)
{ {
my $problem = $anvil->Cluster->parse_cib({debug => 2}); my $problem = $anvil->Cluster->parse_cib({debug => 2});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
if ($problem) if (not $problem)
{
my $node_name = $anvil->data->{cib}{parsed}{'local'}{name};
my $ready = $anvil->data->{cib}{parsed}{data}{node}{$node_name}{node_state}{ready};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { ready => $ready }});
if ($ready)
{
# We're good.
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0279"});
$anvil->Job->update_progress({progress => 15, message => "job_0279"});
}
else
{
# Node isn't ready yet.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0278"});
$anvil->Job->update_progress({progress => 10, message => "job_0278"});
}
}
else
{
# Cluster hasn't started.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0277"});
$anvil->Job->update_progress({progress => 5, message => "job_0277"});
}
if ($waiting)
{ {
# The cluster isn't running, sleep and exit.
$anvil->Job->update_progress({
progress => 0,
message => "error_0233",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0233"});
sleep 10; sleep 10;
$anvil->nice_exit({exit_code => 1});
} }
} }
else
{
# This is not the tool to move to DR.
$anvil->Job->update_progress({
progress => 100,
message => "error_0221",
job_status => "failed",
});
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 0, priority => 'err', key => "error_0221"});
$anvil->nice_exit({exit_code => 1});
}
return(0); return(0);
} }

2
tools/anvil-rename-server
Unescape View File

@ -47,7 +47,7 @@ if (not $anvil->data->{sys}{database}{connections})
{ {
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try
# again after we exit. # again after we exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0077"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"});
sleep 10; sleep 10;
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }

25
tools/anvil-safe-start
Unescape View File

@ -101,6 +101,7 @@ start_pacemaker($anvil);
# Boot servers. # Boot servers.
boot_servers($anvil); boot_servers($anvil);
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "job_0281"});
$anvil->nice_exit({exit_code => 0}); $anvil->nice_exit({exit_code => 0});
@ -113,9 +114,24 @@ sub boot_servers
{ {
my ($anvil) = @_; my ($anvil) = @_;
### TODO: We need to handle boot ordering, once the WebUI is at that stage. For now, bling-boot all # Call 'anvil-boot-server --server all' to boot the servers now.
### servers. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0614"});
my $shell_call = $anvil->data->{path}{exe}{'anvil-boot-server'}." --server all";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
if ($return_code)
{
# What?! Fail out, we're done.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0275", variables => {
output => $output,
return_code => $return_code,
}});
$anvil->nice_exit({exit_code => 1});
}
return(0); return(0);
} }
@ -149,7 +165,8 @@ sub start_pacemaker
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0608"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, priority => "alert", key => "log_0608"});
### TODO: A lot more testing is needed for degraded single-node start later. ### TODO: A lot more testing is needed for degraded single-node start later.
#my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start"; ### Should we use --all, or wait for our peer? For now, we wait.
#my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start --all";
my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start"; my $shell_call = $anvil->data->{path}{exe}{pcs}." cluster start";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call}); my ($output, $return_code) = $anvil->System->call({shell_call => $shell_call});

15
tools/anvil-shutdown-server
Unescape View File

@ -48,7 +48,7 @@ if (not $anvil->data->{sys}{database}{connections})
{ {
# No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try
# again after we exit. # again after we exit.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0077"}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0075"});
sleep 10; sleep 10;
$anvil->nice_exit({exit_code => 1}); $anvil->nice_exit({exit_code => 1});
} }
@ -111,6 +111,15 @@ if ($anvil->data->{switches}{'server-uuid'})
'switches::server' => $anvil->data->{switches}{'server'}, 'switches::server' => $anvil->data->{switches}{'server'},
}}); }});
} }
else
{
# Invalid server UUID.
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0269", variables => {
server_uuid => $anvil->data->{switches}{'server-uuid'},
}});
$anvil->Job->update_progress({progress => 100, message => "error_0269,!!server_uuid!".$anvil->data->{switches}{'server-uuid'}."!!"});
$anvil->nice_exit({exit_code => 1});
}
} }
# Do we have a server name? # Do we have a server name?
@ -152,7 +161,7 @@ if (not $anvil->data->{sys}{anvil_uuid})
# being up. # being up.
wait_for_pacemaker($anvil); wait_for_pacemaker($anvil);
# If 'server' is 'all', boot all servers. # If 'server' is 'all', shut down all servers.
if (lc($anvil->data->{switches}{'server'}) eq "all") if (lc($anvil->data->{switches}{'server'}) eq "all")
{ {
shutdown_all_servers($anvil); shutdown_all_servers($anvil);
@ -177,7 +186,7 @@ sub wait_for_pacemaker
{ {
my ($anvil) = @_; my ($anvil) = @_;
# Boot the server using pcs, but of course, wait for the node to be up. # Shutdown the server using pcs, but of course, wait for the node to be up.
my $waiting = 1; my $waiting = 1;
while($waiting) while($waiting)
{ {

Write Preview
Loading…
Cancel
Save