my $definition_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server.".xml";
my $definition_file = $anvil->data->{path}{directories}{shared}{definitions}."/".$server.".xml";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { definition_file => $definition_file }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { definition_file => $definition_file }});
@ -359,7 +357,11 @@ sub start_server
{
{
# If this fails, we want to exit with OCF_ERR_CONFIGURED (6) so that pacemaker doesn't try to
# If this fails, we want to exit with OCF_ERR_CONFIGURED (6) so that pacemaker doesn't try to
# also start the server on another node, because we don't know the state of it here.
# also start the server on another node, because we don't know the state of it here.
to_log($anvil, {message => "All tests passed, yet the attempt to boot the server: [$server] exited with a non-zero return code: [$return_code]. The server is in an unknown state, so exiting with a fatal error. Human intervention is now required. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
# If this fails, we want to exit with OCF_ERR_CONFIGURED (6) so that pacemaker doesn't try to
# If this fails, we want to exit with OCF_ERR_CONFIGURED (6) so that pacemaker doesn't try to
# also start the server on another node, because we don't know the state of it here.
# also start the server on another node, because we don't know the state of it here.
to_log($anvil, {message => "It appears that the call to boot the server: [$server] worked, but the call to list running servers exited with a non-zero return code: [$return_code]. The server is in an unknown state, so exiting with a fatal error. Human intervention is now required. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "The server: [$server] should have been started, but it's state is: [$state]. Human intervention is required!", 'line' => __LINE__, level => 1, priority => "err"});
# If we're still alive, then we didn't see the server in the list of running servers, which is really weird.
# If we're still alive, then we didn't see the server in the list of running servers, which is really weird.
to_log($anvil, {message => "The server: [$server] should have been started, but it wasn't found in the list of running servers.", 'line' => __LINE__, level => 1, priority => "err"});
to_log($anvil, {message => "The attempt to list the running servers returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
# The server is paused. Resume it, wait a few, then proceed with the shutdown.
# The server is paused. Resume it, wait a few, then proceed with the shutdown.
to_log($anvil, {message => "The server: [$server] is paused. Resuming it now so that it can react to the shutdown request.", 'line' => __LINE__, level => 2});
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." resume $server"});
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{virsh}." resume $server"});
if ($return_code)
if ($return_code)
{
{
# Looks like virsh isn't running.
# Looks like virsh isn't running.
to_log($anvil, {message => "The attempt to resume the server: [$server] returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "Pausing for a moment to give the server time to resume.", 'line' => __LINE__, level => 2});
to_log($anvil, {message => "Pausing for a moment to give the server time to resume.", 'line' => __LINE__, level => 2});
@ -470,7 +480,11 @@ sub stop_server
if ($return_code)
if ($return_code)
{
{
# Looks like virsh isn't running.
# Looks like virsh isn't running.
to_log($anvil, {message => "The attempt to wake the server: [$server] returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "Pausing for half a minute to give the server time to wake up.", 'line' => __LINE__, level => 2});
to_log($anvil, {message => "Pausing for half a minute to give the server time to wake up.", 'line' => __LINE__, level => 2});
@ -496,7 +510,11 @@ sub stop_server
if ($return_code)
if ($return_code)
{
{
# Looks like virsh isn't running.
# Looks like virsh isn't running.
to_log($anvil, {message => "The attempt to force-off the server: [$server] returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "The server: [$server] is running, but it is in an unexpected state: [$state]. Human intervention is required!", 'line' => __LINE__, level => 1, priority => "err"});
to_log($anvil, {message => "The attempt to shut down the server: [$server] returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "The attempt to list the running servers returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
# If I got a non-zero return code, something went wrong with the virsh call.
# If I got a non-zero return code, something went wrong with the virsh call.
if ($return_code)
if ($return_code)
{
{
to_log($anvil, {message => "It would appear that libvirtd is not operating (or not operating correctly). Expected the return code '0' but got: [$return_code].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "It appears that the call to check if the server: [$server] is on this node returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
to_log($anvil, {message => "It appears that the call to check if the server: [$server] is on this node returned a non-zero return code: [$return_code]. The output, if any, was: [$output].", 'line' => __LINE__, level => 0, priority => "err"});
$anvil->nice_exit({exit_code => 1});
$anvil->nice_exit({exit_code => 1});
}
}
@ -1521,7 +1563,7 @@ sub validate_emulator
# What emulator is this using?
# What emulator is this using?
my $emulator = $anvil->data->{server}{definition_xml}->{devices}->[0]->{emulator}->[0];
my $emulator = $anvil->data->{server}{definition_xml}->{devices}->[0]->{emulator}->[0];
<keyname="log_0302">We were invoked with an unexpected (or no) command. Environment variables and arguments below.</key>
<keyname="log_0302">We were invoked with an unexpected (or no) command. Environment variables and arguments below.</key>
<keyname="log_0303">We've been asked to start the server: [#!variable!server!#].</key>
<keyname="log_0303">We've been asked to start the server: [#!variable!server!#].</key>
<keyname="log_0304">It appears that the list the currently running servers returned a non-zero return code: [#!variable!return_code!#]. We will proceed as we may be able to fix this. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0304">It appears that the list the currently running servers returned a non-zero return code: [#!variable!return_code!#]. We will proceed as we may be able to fix this. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0305">#!free!#</key>
<keyname="log_0305">Sanity checks passed, ready to start: [#!variable!server!#].</key>
<keyname="log_0306">The server: [#!variable!server!#] is already on this node in the state: [#!variable!state!#], aborting the start request.</key>
<keyname="log_0306">The server: [#!variable!server!#] is already on this node in the state: [#!variable!state!#], aborting the start request.</key>
<keyname="log_0307"></key>
<keyname="log_0307">All tests passed, yet the attempt to boot the server: [#!variable!server!#] exited with a non-zero return code: [#!variable!return_code!#]. The server is in an unknown state, so exiting with a fatal error. Human intervention is now required. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0308"></key>
<keyname="log_0308">It appears that the call to boot the server: [#!variable!server!#] worked, but the call to list running servers exited with a non-zero return code: [#!variable!return_code!#]. The server is in an unknown state, so exiting with a fatal error. Human intervention is now required. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0309"></key>
<keyname="log_0309">The server: [#!variable!server!#] has started successfully.</key>
<keyname="log_0310"></key>
<keyname="log_0310">The server: [#!variable!server!#] should have been started, but it's state is: [#!variable!state!#]. Human intervention is required!</key>
<keyname="log_0311"></key>
<keyname="log_0311">The server: [#!variable!server!#] should have been started, but it wasn't found in the list of running servers.</key>
<keyname="log_0312"></key>
<keyname="log_0312">The attempt to list the running servers returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0313"></key>
<keyname="log_0313">The server: [#!variable!server!#] is running. We will ask it to shut down now.</key>
<keyname="log_0314"></key>
<keyname="log_0314">The server: [#!variable!server!#] is paused. Resuming it now so that it can react to the shutdown request.</key>
<keyname="log_0315"></key>
<keyname="log_0315">The attempt to resume the server: [#!variable!server!#] returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0316"></key>
<keyname="log_0316">Pausing for a moment to give the server time to resume.</key>
<keyname="log_0317"></key>
<keyname="log_0317">The server: [#!variable!server!#] is asleep. Waking it now so that it can react to the shutdown request.</key>
<keyname="log_0318"></key>
<keyname="log_0318">The attempt to wake the server: [#!variable!server!#] returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0319"></key>
<keyname="log_0319">Pausing for half a minute to give the server time to wake up.</key>
<keyname="log_0320"></key>
<keyname="log_0320">The server: [#!variable!server!#] is already shutting down. We'll monitor it until it actually shuts off.</key>
<keyname="log_0321">The server: [#!variable!server!#] is already off.</key>
<keyname="log_0322">The server: [#!variable!server!#] is hung. Its state is: [#!variable!state!#]. We will force it off now.</key>
<keyname="log_0323">The attempt to force-off the server: [#!variable!server!#] returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0324">The server: [#!variable!server!#] is now off.</key>
<keyname="log_0325">The server: [#!variable!server!#] is running, but it is in an unexpected state: [#!variable!state!#]. Human intervention is required!</key>
<keyname="log_0326">The server: [#!variable!server!#] was not listed on this node, so it is not running here.</key>
<keyname="log_0327">Asking the server: [#!variable!server!#] to shut down now. Please be patient.</key>
<keyname="log_0328">The attempt to shut down the server: [#!variable!server!#] returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0329">The server: [#!variable!server!#] is no longer listed. It is now off.</key>
<keyname="log_0330">The server: [#!variable!server!#] is not off yet, waiting a few seconds and then we'll check again.</key>
<keyname="log_0331">The environment variable 'OCF_RESKEY_CRM_meta_timeout' was not set, so setting it to: [#!variable!timeout!#].</key>
<keyname="log_0332">The 'virsh' call exited with the return code: [#!variable!return_code!#]. The 'libvirtd' may have failed to start. We won't wait any longer.</key>
<keyname="log_0333">The 'virsh' call exited with the return code: [#!variable!return_code!#]. The 'libvirtd' service might be starting, so we will check again shortly.</key>
<keyname="log_0334">It would appear that libvirtd is not operating (or not operating correctly). Expected the return code '0' but got: [#!variable!return_code!#].</key>
<keyname="log_0335">
Output of: [#!variable!command!#] was;
==========
#!variable!output!#
==========
</key>
<keyname="log_0336">The server: [#!variable!server!#] is: [#!variable!state!#], which is OK.</key>
<keyname="log_0337">The server: [#!variable!server!#] is: [#!variable!state!#].</key>
<keyname="log_0338">The server: [#!variable!server!#] is in a bad state: [#!variable!state!#]!</key>
<keyname="log_0339">The server: [#!variable!server!#] is in an unexpected state: [#!variable!state!#]!</key>
<keyname="log_0340">The server: [#!variable!server!#] is not running on this node.</key>
<keyname="log_0341">We're pushing the: [#!variable!server!#] to: [#!variable!target!#].</key>
<keyname="log_0342">It appears that the call to check if the server: [#!variable!server!#] is on this node returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0343">The server: [#!variable!server!#] state is: [#!variable!state!#]. A server must be 'running' in order to migrate it.</key>
<keyname="log_0344">The server: [#!variable!server!#] wasn't found on this machine.</key>
<keyname="log_0345">Verifying that the server: [#!variable!server!#] was successfully migrated here.</key>
<keyname="log_0346">While verifying that the server: [#!variable!server!#] migrated here, the attempt to list servers running here returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0347">The migration of the server: [#!variable!server!#] to here was successful!</key>
<keyname="log_0348">It looks like we were called to verify that the: [#!variable!server!#] migrated here, but it isn't here yet. We'll proceed with an attempt to pull the server over.</key>
<keyname="log_0350">Temporarily enabling dual primary for the resource: [#!variable!resource!#] to the node: [#!variable!target_name!# (#!variable!target_node_id!#].</key>
<keyname="log_0351">The attempt to enable dual-primary for the resource: [#!variable!resource!#] to the node: [#!variable!target_name!# (#!variable!target_node_id!#)] returned a non-zero return code [#!variable!return_code!#]. The returned output (if any) was: [#!variable!output!#].</key>
<keyname="log_0352">The migration of: [#!variable!server!#] to the node: [#!variable!target!#] will now begin.</key>
<keyname="log_0353">The attempt to migrate the server: [#!variable!server!#] to the node: [#!variable!target!#] returned a non-zero return code [#!variable!return_code!#]. The returned output (if any) was: [#!variable!output!#].</key>
<keyname="log_0354">It looks like the migration was successful. Will verify in a moment.</key>
<keyname="log_0355">Re-disabling dual primary by restoring config file settings.</key>
<keyname="log_0356">The attempt to reset DRBD to config file settings returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0357">Failure, exiting with '1'.</key>
<keyname="log_0358">It appears that the list the running servers on the migration target: [#!variable!target!#] returned a non-zero return code: [#!variable!return_code!#]. The output, if any, was: [#!variable!output!#].</key>
<keyname="log_0359">The migration of the server: [#!variable!server!#] to: [#!variable!target!#] was a success!</key>
<keyname="log_0360">Success, exiting with '0'.</key>
<keyname="log_0362">- Server definition was read.</key>
<keyname="log_0363">- Server name is valid.</key>
<keyname="log_0364">- Eumlator is valid.</key>
<keyname="log_0365">- Sufficient RAM is available.</key>
<keyname="log_0366">- Network bridge(s) are available.</key>
<keyname="log_0367">- Storage is valid and ready.</key>
<keyname="log_0368">The bridge: [#!variable!bridge!#] is available for this server.</key>
<keyname="log_0369">The server wants to connect to the bridge: [#!variable!bridge!#] which we do not have on this node.</key>
<keyname="log_0370">The attempt to read the DRBD configuration returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!drbd_body!#].</key>
<keyname="log_0371">Recording the local connection details for the resource: [#!variable!resource!#] -> [#!variable!address!#:#!variable!port!#].</key>
<keyname="log_0372">Recording the peer's connection details for the resource: [#!variable!resource!#] -> [#!variable!address!#:#!variable!port!#].</key>
<keyname="log_0373">Checking that the DRBD device: [#!variable!device_path!#] is ready.</key>
<keyname="log_0374">The server wants to use: [#!variable!device_path!#] as a hard drive, but we couldn't find the backing logical volume on this node.</key>
<keyname="log_0375">The server wants to use: [#!variable!device_path!#] as a hard drive, but the backing logical volume: [#!variable!lv!#] doesn't exist on this node.</key>
<keyname="log_0376">The server wants to use: [#!variable!device_path!#] as a hard drive, which is backed by the logical volume: [#!variable!lv!#]. Checking that these are ready.</key>
<keyname="log_0377">The attempt to read the DRBD status returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!status_json!#].</key>
<keyname="log_0378">DRBD is not loaded. Bringing it up now.</key>
<keyname="log_0379">Bringing up the resource: [#!variable!resource!#] for the server's: [".#!variable!device_path!#."] disk.</key>
<keyname="log_0380">The attempt to start the DRBD resource: [#!variable!resource!#] returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!output!#].</key>
<keyname="log_0381">Pausing briefly to give the resources time to start.</key>
<keyname="log_0382">The attempt to read the DRBD status after bringing up the resource(s) for this server returned a non-zero code: [#!variable!return_code!#]. The returned output (if any) was: [#!variable!status_json!#].</key>
<keyname="log_0383">The attempt to read the DRBD status after bringing up the resource(s) appears to have failed.</key>
<keyname="log_0384">The DRBD resource: [#!variable!resource!#] backing the device: [#!variable!device_path!#] was not seen in the 'drbdsetup' status data. Attempting to bringing it up now.</key>
<keyname="log_0385">Checking the DRBD status again.</key>
<keyname="log_0386">The DRBD resource: [#!variable!resource!#] backing the device: [#!variable!device_path!#] was not able to start.</key>
<keyname="log_0387">Checking that the peer's DRBD resources are Connected and UpToDate prior to migration.</key>
<keyname="log_0388">The local replicated disk: [#!variable!device_path!#] is used by this server. Checking it out now.</key>
<keyname="log_0389">The DRBD resource: [#!variable!resource!#] volume: [#!variable!volume!#] locat disk state is: [#!variable!disk_state!#]. Unsafe to boot the server unless the disk state is UpToDate.</key>
<keyname="log_0390">The DRBD resource: [#!variable!resource!#] volume: [#!variable!volume!#] locat disk state is: [#!variable!disk_state!#], good.</key>
<keyname="log_0391">Checking connection to: [#!variable!name!#].</key>
<keyname="log_0392">The DRBD resource: [#!variable!resource!#] on the peer: [#!variable!name!#] is 'Primary'. Refusing to boot.</key>
<keyname="log_0394">Ignoring the connection to: [#!variable!peer_short_name!#], it isn't the migration target.</key>
<keyname="log_0395">The DRBD resource: [#!variable!resource!#] on the peer: [#!variable!name!#] is not UpToDate (or SyncSource). Refusing to migrate.</key>
<keyname="log_0396">Ignoring the local replicated disk: [#!variable!device_path!#], it is not used by this server.</key>
<keyname="log_0397">Checking that the optical disc image: [#!variable!file!#] exists.</key>
<keyname="log_0398">The server has the ISO: [#!variable!file!#] mounted in its optical drive, but that file doesn't exist on this system.</key>
<keyname="log_0399">The server has the ISO: [#!variable!file!#] mounted in its optical drive, which we have, but we can't read it. Check permissions and for SELinux denials.</key>
<keyname="log_0400">The server has the ISO: [#!variable!file!#] mounted in its optical drive, which we have.</key>
<keyname="log_0401">The server wants to use the emulator: [#!variable!emulator!#] which doesn't exist on this node. Was this server migrated from a different generation Anvil! system? Please update '<emulator>...</emulator>' in the server's definition file: [#!variable!definition_file!#].</key>
<keyname="log_0402">The server wants to use the emulator: [#!variable!emulator!#] which exists, but we can't run. Please check permissions and for SELinux denials.</key>
<keyname="log_0403">The configured server name: [#!variable!server!#] does not match the name of the server in the definition file: [#!variable!name!#]!</key>
<keyname="log_0404">The configured server name: [#!variable!name!#] needs: [#!variable!ram!# (#!variable!ram_bytes!# bytes)] of RAM, but only: #!variable!available_ram!# (#!variable!available_ram_bytes!# bytes)] are available!</key>
<keyname="log_0405">The definition file: [#!variable!definition_file!#] for the server: [#!variable!server!#] does not exist here!</key>
<keyname="log_0406">The definition file: [#!variable!definition_file!#] for the server: [#!variable!server!#] can not be read!</key>
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->
<!-- Test words. Do NOT change unless you update 't/Words.t' or tests will needlessly fail. -->