seasharp
/
anvil
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Continuing work on anvil-manage-dr. Got it to the point where it should (but doesn't yet) create the new DRBD config and the LV(s) on DR.

Browse Source 
Signed-off-by: Digimer <digimer@alteeve.ca>
main
Digimer 3 years ago
parent 5b35204af4
commit 20a784baa2
2 changed files with 329 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      Anvil/Tools/Storage.pm
  2. 410
      tools/anvil-manage-dr

4
Anvil/Tools/Storage.pm
Unescape View File

@ -4955,9 +4955,9 @@ fi";
my $shell_call = " my $shell_call = "
if [ -d '".$directory."' ]; if [ -d '".$directory."' ];
then then
".$anvil->data->{path}{exe}{echo}." 'exists'; echo 'exists';
else else
".$anvil->data->{path}{exe}{echo}." 'not found'; echo 'not found';
fi"; fi";
$anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0166", variables => { shell_call => $shell_call, target => $target, remote_user => $remote_user }}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => $debug, key => "log_0166", variables => { shell_call => $shell_call, target => $target, remote_user => $remote_user }});
(my $output, $error, my $return_code) = $anvil->Remote->call({ (my $output, $error, my $return_code) = $anvil->Remote->call({

410
tools/anvil-manage-dr
Unescape View File

@ -15,6 +15,7 @@ use warnings;
use Anvil::Tools; use Anvil::Tools;
require POSIX; require POSIX;
use Term::Cap; use Term::Cap;
use Text::Diff;
my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0];
my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0];
@ -122,7 +123,7 @@ sub sanity_check
# Get the Anvil! details. # Get the Anvil! details.
$anvil->Database->get_hosts(); $anvil->Database->get_hosts();
$anvil->Database->get_anvils(); $anvil->Database->get_anvils();
$anvil->Database->get_storage_group_data({debug => 2}); $anvil->Database->get_storage_group_data();
# Does this Anvil! have a DR node? # Does this Anvil! have a DR node?
if (not $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid}) if (not $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_dr1_host_uuid})
@ -385,6 +386,7 @@ sub process_protect
# Parse out the DRBD resource's backing the server and get their LV sizes. # Parse out the DRBD resource's backing the server and get their LV sizes.
$anvil->Database->get_server_definitions(); $anvil->Database->get_server_definitions();
my $anvil_uuid = $anvil->Cluster->get_anvil_uuid(); my $anvil_uuid = $anvil->Cluster->get_anvil_uuid();
my $anvil_password = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_password};
my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid}; my $node1_host_uuid = $anvil->data->{anvils}{anvil_uuid}{$anvil_uuid}{anvil_node1_host_uuid};
my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name}; my $node1_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{host_name};
my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name}; my $node1_short_host_name = $anvil->data->{hosts}{host_uuid}{$node1_host_uuid}{short_host_name};
@ -400,6 +402,7 @@ sub process_protect
my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml}; my $server_definition_xml = $anvil->data->{server_definitions}{server_definition_server_uuid}{$server_uuid}{server_definition_xml};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
anvil_uuid => $anvil_uuid, anvil_uuid => $anvil_uuid,
anvil_password => $anvil->Log->is_secure($anvil_password),
node1_host_uuid => $node1_host_uuid, node1_host_uuid => $node1_host_uuid,
node1_host_name => $node1_host_name, node1_host_name => $node1_host_name,
node1_short_host_name => $node1_short_host_name, node1_short_host_name => $node1_short_host_name,
@ -423,7 +426,7 @@ sub process_protect
definition => $server_definition_xml, definition => $server_definition_xml,
}); });
$anvil->DRBD->gather_data(); $anvil->DRBD->gather_data({debug => 2});
my $server_ram = $anvil->data->{server}{$short_host_name}{$server_name}{'from_db'}{memory}; my $server_ram = $anvil->data->{server}{$short_host_name}{$server_name}{'from_db'}{memory};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
@ -449,13 +452,16 @@ sub process_protect
my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk}; my $backing_disk = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{backing_disk};
my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor}; my $device_minor = $anvil->data->{new}{resource}{$resource}{host}{$this_host_name}{volume}{$volume}{device_minor};
my $tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port}; my $tcp_port = $anvil->data->{new}{resource}{$resource}{peer}{$this_host_name}{tcp_port};
my $this_size = $anvil->Storage->get_size_of_block_device({host_uuid => $this_host_uuid, path => $backing_disk});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
"s1:volume" => $volume, "s1:volume" => $volume,
"s2:device_path" => $device_path, "s2:device_path" => $device_path,
"s3:backing_disk" => $backing_disk, "s3:backing_disk" => $backing_disk,
"s4:device_minor" => $device_minor, "s4:device_minor" => $device_minor,
"s5:this_size" => $anvil->Convert->add_commas({number => $this_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_size}).")", }});
my $this_size = $anvil->Storage->get_size_of_block_device({debug => 2, host_uuid => $this_host_uuid, path => $backing_disk});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
this_size => $anvil->Convert->add_commas({number => $this_size})." (".$anvil->Convert->bytes_to_human_readable({'bytes' => $this_size}).")",
}}); }});
if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{size}) or (not $anvil->data->{server}{drbd}{$resource}{$volume}{size})) if ((not exists $anvil->data->{server}{drbd}{$resource}{$volume}{size}) or (not $anvil->data->{server}{drbd}{$resource}{$volume}{size}))
@ -878,120 +884,328 @@ sub process_protect
}}); }});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_resource_config => $new_resource_config }});
=cut # Is the new res file the same as the old one?
# Server srv02-c8-b, example showing two disks in one VM. my $difference = diff \$old_resource_config, \$new_resource_config, { STYLE => 'Unified' };
resource srv02-c8-b { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { difference => $difference }});
on an-a01n01 {
node-id 0;
volume 0 {
device /dev/drbd_srv02-c8-b_0 minor 0;
disk /dev/rhel/srv02-c8-b_0;
meta-disk internal;
}
volume 1 {
device /dev/drbd_srv02-c8-b_1 minor 1;
disk /dev/rhel/srv02-c8-b_1;
meta-disk internal;
}
}
on an-a01n02 { if (not $difference)
node-id 1; {
volume 0 { print "The resource file: [".$config_file."] doesn't need to be updated.\n";
device /dev/drbd_srv02-c8-b_0 minor 0;
disk /dev/rhel/srv02-c8-b_0;
meta-disk internal;
}
volume 1 {
device /dev/drbd_srv02-c8-b_1 minor 1;
disk /dev/rhel/srv02-c8-b_1;
meta-disk internal;
}
} }
else
{
# Write out a test file.
my $test_file = $anvil->data->{path}{directories}{temp}."/test-".$server_name.".res";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { test_file => $test_file }});
my ($problem) = $anvil->Storage->write_file({
debug => 2,
backup => 0,
overwrite => 1,
file => $test_file,
body => $new_resource_config,
user => "root",
group => "root",
mode => "0644",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
on an-a01dr01 { # Validate.
node-id 2; my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --config-to-test ".$test_file." --config-to-exclude ".$config_file." sh-nop";
volume 0 { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
device /dev/drbd_srv02-c8-b_0 minor 0; my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." dump-xml"});
disk /dev/rhel_new-dr/srv02-c8-b_0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
meta-disk internal; output => $output,
} return_code => $return_code,
volume 1 { }});
device /dev/drbd_srv02-c8-b_1 minor 1; if ($return_code)
disk /dev/rhel_new-dr/srv02-c8-b_1; {
meta-disk internal; # Something went wrong.
} print "[ Error ] - The check appears to have failed. Expected a return code of '0', but got: [".$return_code."]\n";
print "The output, if any, was:\n";
print "====\n";
print $output."\n";
print "====\n";
$anvil->nice_exit({exit_code => 1});
} }
### NOTE: Remember to open the appropriate firewall port! # Remove the test file.
# firewall-cmd --zone=SN1 --permanent --add-port=7788/tcp --permanent unlink $test_file;
# firewall-cmd --zone=SN1 --permanent --add-port=7788/tcp
connection { # Backup the res file so we can tell the user where the current config was backed up to in
host an-a01n01 address 10.101.12.1:7788; # case they need to restore it.
host an-a01n02 address 10.101.12.2:7788; my ($backup_file) = $anvil->Storage->backup({file => $config_file});
net { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
protocol C; print "Backed up old config as: [".$backup_file."]. Updating it now.\n";
fencing resource-and-stonith;
} # Write out the new file.
} ($problem) = $anvil->Storage->write_file({
connection { debug => 2,
host an-a01n01 address 10.101.12.1:7789; backup => 0,
host an-a01dr01 address 10.101.12.3:7789; overwrite => 1,
net { file => $config_file,
protocol A; body => $new_resource_config,
fencing dont-care; user => "root",
group => "root",
mode => "0644",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
print "Updated! Verifying...\n";
# Call 'drbdadm dump-xml' to check that it's OK.
($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." dump-xml"});
if ($return_code)
{
# Something went wrong.
print "[ Error ] - The check appears to have failed. Expected a return code of '0', but got: [".$return_code."]\n";
print "The output, if any, was:\n";
print "====\n";
print $output."\n";
print "====\n";
print "- Restoring the old config now.\n";
my ($backup_file) = $anvil->Storage->backup({file => $config_file});
print "- The problematic new config has been saved as: [".$backup_file."]\n";
# Write out the new file.
my ($problem) = $anvil->Storage->write_file({
debug => 2,
backup => 1,
overwrite => 1,
file => $old_resource_config,
body => $config_file,
user => "root",
group => "root",
mode => "0644",
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { backup_file => $backup_file }});
print "- The old config has been restored. Exiting.\n";
$anvil->nice_exit({exit_code => 1});
} }
# New config is good! Update the file on the peers.
print "- The new config looks good!\n";
} }
connection {
host an-a01n02 address 10.101.12.2:7790; # New config is good! Update the file on the peers.
host an-a01dr01 address 10.101.12.3:7790; print "- Updating the peers now...\n";
net { foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
protocol A; {
fencing dont-care; # "Peer" in this context is either a node or a DR host
next if $this_host_uuid eq $anvil->Get->host_uuid();
my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
my $peer_sn_ip = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{sn1}{ip_address};
print "- Updating the resource file: [".$config_file."] on the host: [".$peer_host_name."] via IP: [".$peer_sn_ip."]\n";
my ($problem) = $anvil->Storage->write_file({
debug => 2,
backup => 1,
overwrite => 1,
file => $new_resource_config,
body => $config_file,
user => "root",
group => "root",
mode => "0644",
target => $peer_sn_ip,
password => $anvil_password,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }});
} }
# Create the LV and MD on DR.
print "Creating logical volumes on DR, if needed. New LVs will have metadata created.\n";
my $create_md = 0;
foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{server}{drbd}{$server_name}})
{
print "- Volume: [".$volume."]\n";
my $dr1_sn1_ip = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{network}{sn1}{ip_address};
my $lv_path = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lv_path};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
volume => $volume,
dr1_sn1_ip => $dr1_sn1_ip,
lv_path => $lv_path,
}});
my $lv_check_call = "
if [ -e '".$lv_path."' ];
then
echo exists;
else
echo create;
fi";
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lv_check_call => $lv_check_call }});
my ($output, $error, $return_code) = $anvil->Remote->call({
target => $dr1_sn1_ip,
password => $anvil_password,
shell_call => $lv_check_call,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
if ($output eq "exists")
{
print "- The logical volume: [".$lv_path."] already exists, skipping it, and NOT create DRBD meta data.\n";
next;
} }
}
==== # Create the LV.
# Resource for srv02-c8-b my $lvcreate_call = $anvil->data->{server}{dr}{volumes}{$server_name}{$volume}{lvcreate_call};
resource srv02-c8-b { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { lvcreate_call => $lvcreate_call }});
on an-a01n01 { ($output, $error, $return_code) = $anvil->Remote->call({
node-id 0; target => $dr1_sn1_ip,
volume 0 { password => $anvil_password,
device /dev/drbd_srv02-c8-b_0 minor 1; shell_call => $lvcreate_call,
disk /dev/cs_an-a01n01/srv02-c8-b_0; });
meta-disk internal; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
print "- LV create call return code: [".$return_code."], output:\n";
print "====\n";
print $output."\n";
print "====\n";
sleep 1;
# Does it exist now?
($output, $error, $return_code) = $anvil->Remote->call({
target => $dr1_sn1_ip,
password => $anvil_password,
shell_call => $lv_check_call,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
if ($output eq "create")
{
print "- The logical volume: [".$lv_path."] creation failed. Unable to proceed.\n";
$anvil->nice_exit({exit_code => 1});
} }
# Create the DRBD meta data now.
$create_md = 1;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { create_md => $create_md }});
} }
on an-a01n02 { if ($create_md)
node-id 1; {
volume 0 { my $dr1_sn1_ip = $anvil->data->{hosts}{host_uuid}{$dr1_host_uuid}{network}{sn1}{ip_address};
device /dev/drbd_srv02-c8-b_0 minor 1; my $drbd_md_call = $anvil->data->{path}{exe}{drbdadm}." --force create-md --max-peers=3 ".$server_name;
disk /dev/cs_an-a01n02/srv02-c8-b_0; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
meta-disk internal; dr1_sn1_ip => $dr1_sn1_ip,
drbd_md_call => $drbd_md_call,
}});
my ($output, $error, $return_code) = $anvil->Remote->call({
target => $dr1_sn1_ip,
password => $anvil_password,
shell_call => $drbd_md_call,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
# Get the VG name that this volume will be created on.
print " - The DRBD metadata creation call return code: [".$return_code."], output:\n";
print "====\n";
print $output."\n";
print "====\n";
} }
# Reload the config.
my $shell_call = $anvil->data->{path}{exe}{drbdadm}." --adjust ".$server_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { shell_call => $shell_call }});
my ($output, $return_code) = $anvil->System->call({shell_call => $anvil->data->{path}{exe}{drbdadm}." dump-xml"});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
output => $output,
return_code => $return_code,
}});
foreach my $this_host_uuid ($node1_host_uuid, $node2_host_uuid, $dr1_host_uuid)
{
# "Peer" in this context is either a node or a DR host
next if $this_host_uuid eq $anvil->Get->host_uuid();
my $peer_host_name = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{short_host_name};
my $peer_sn_ip = $anvil->data->{hosts}{host_uuid}{$this_host_uuid}{network}{sn1}{ip_address};
print "- Reloading the resource: [".$server_name."] on the host: [".$peer_host_name."]\n";
my ($output, $error, $return_code) = $anvil->Remote->call({
target => $peer_sn_ip,
password => $anvil_password,
shell_call => $shell_call,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
} }
### NOTE: Remember to open the appropriate firewall port! # Log into DR and up the resource.
# firewall-cmd --zone=SN1 --permanent --add-port=7789/tcp --permanent print "- Asking the DR host to bring up the: [".$server_name."] resource now...\n";
# firewall-cmd --zone=SN1 --permanent --add-port=7789/tcp my $drbd_md_call = $anvil->data->{path}{exe}{drbdadm}." up ".$server_name;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { drbd_md_call => $drbd_md_call }});
($output, my $error, $return_code) = $anvil->Remote->call({
target => $dr1_sn1_ip,
password => $anvil_password,
shell_call => $drbd_md_call,
});
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
error => $error,
output => $output,
return_code => $return_code,
}});
# Get the VG name that this volume will be created on.
print " - The resource up command's return code was: [".$return_code."], output:\n";
print "====\n";
print $output."\n";
print "====\n";
# Now watch until the DR host shows up
print "Checking to see if the DR host has connected to this resource yet.\n";
my $waiting = 1;
while($waiting)
{
sleep 5;
$anvil->DRBD->gather_data({debug => 2});
connection { my $dr_seen = 1;
host an-a01n01 address 10.101.10.1:7789; foreach my $volume (sort {$a cmp $b} keys %{$anvil->data->{new}{resource}{$server_name}{volume}})
host an-a01n02 address 10.101.10.2:7789; {
disk { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { volume => $volume }});
# Without this, the variable bit rate caps at 100 MiB/sec, and most deployments are if (exists $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name})
# 10 Gbps. So this lets the variable rate climb to 500 MiB/sec {
c-max-rate 500M; my $local_role = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{local_role};
my $local_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{local_disk_state};
my $peer_role = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{peer_role};
my $peer_disk_state = $anvil->data->{new}{resource}{$server_name}{volume}{$volume}{peer}{$dr1_short_host_name}{peer_disk_state};
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => {
local_role => $local_role,
local_disk_state => $local_disk_state,
peer_role => $peer_role,
peer_disk_state => $peer_disk_state,
}});
} }
net { else
protocol C; {
fencing resource-and-stonith; # Not up yet.
print "- Not up yet, will check again in five seconds.\n";
$dr_seen = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { dr_seen => $dr_seen }});
}
}
if ($dr_seen)
{
# We're ready.
print "- Up!\n";
$waiting = 0;
$anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { waiting => $waiting }});
} }
} }
}
=cut
return(0); return(0);
} }

Write Preview
Loading…
Cancel
Save