#!/usr/bin/perl # # This program will manage servers; Changing RAM, CPU cores, Growing virtual disks, adding virtual disks, # inserting and ejecting ISO images into virtual optical media. # # Exit codes; # 0 = Normal exit. # 1 = No database connection. # # TODO: # # USAGE: # - Show # - anvil-manage-server-storage --server srv01-fs37 # use strict; use warnings; use Anvil::Tools; require POSIX; use Term::Cap; use Text::Diff; use Data::Dumper; use Sys::Virt; my $THIS_FILE = ($0 =~ /^.*\/(.*)$/)[0]; my $running_directory = ($0 =~ /^(.*?)\/$THIS_FILE$/)[0]; if (($running_directory =~ /^\./) && ($ENV{PWD})) { $running_directory =~ s/^\./$ENV{PWD}/; } # Turn off buffering so that the pinwheel will display while waiting for the SSH call(s) to complete. $| = 1; my $anvil = Anvil::Tools->new(); # Read switches (target ([user@]host[:port]) and the file with the target's password. $anvil->Get->switches({list => [ "add", "anvil", "boot", "boot-menu", "confirm", "disk", "eject", "job-uuid", "grow", "insert", "optical", "server", "storage-group", ], man => $THIS_FILE}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => $anvil->data->{switches}}); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 2, key => "log_0115", variables => { program => $THIS_FILE }}); # Connect to the database(s). If we have no connections, we'll proceed anyway as one of the 'run_once' tasks # is to setup the database server. $anvil->Database->connect(); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 3, secure => 0, key => "log_0132"}); if (not $anvil->data->{sys}{database}{connections}) { # No databases, update the job, sleep for a bit and then exit. The daemon will pick it up and try # again after we exit. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0305"}); sleep 10; $anvil->nice_exit({exit_code => 1}); } if ($anvil->data->{switches}{'job-uuid'}) { load_job($anvil); } $anvil->Database->get_hosts(); $anvil->Database->get_anvils(); $anvil->Database->get_servers(); if ($anvil->data->{switches}{anvil}) { # Make sure they asked for a real anvil. $anvil->Get->anvil_from_switch({string => $anvil->data->{switches}{anvil}}); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::anvil_name" => $anvil->data->{switches}{anvil_name}, "switches::anvil_uuid" => $anvil->data->{switches}{anvil_uuid}, }}); } if (not $anvil->data->{switches}{server}) { # Show the list of servers. show_server_list($anvil); $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0456"}); $anvil->Job->update_progress({ progress => 100, message => "error_0456", }) if $anvil->data->{switches}{'job-uuid'}; $anvil->nice_exit({exit_code => 0}); } validate_server($anvil); if ($anvil->data->{switches}{cpu}) { #manage_cpu($anvil); } elsif ($anvil->data->{switches}{ram}) { #manage_ram($anvil); } elsif ($anvil->data->{switches}{boot}) { #manage_boot($anvil); } elsif ($anvil->data->{switches}{'boot-menu'}) { manage_boot_menu($anvil); } else { show_server_details($anvil, 1); } $anvil->Job->update_progress({ progress => 100, message => "job_0281", }) if $anvil->data->{switches}{'job-uuid'}; $anvil->nice_exit({exit_code => 0}); ############################################################################################################# # Functions # ############################################################################################################# sub connect_to_libvirt { my ($anvil) = @_; my $short_host_name = $anvil->Get->short_host_name; my $host_uuid = $anvil->Get->host_uuid; my $server_name = $anvil->data->{switches}{server_name}; my $server_uuid = $anvil->data->{switches}{server_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:short_host_name' => $short_host_name, 's2:host_uuid' => $host_uuid, 's3:server_name' => $server_name, 's4:server_uuid' => $server_uuid, }}); my $from_source = get_definition_source($anvil); my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source, server_state => $server_state, }}); # We'll make changes to the DB and on-disk definitions if the server isn't running. If it is, we'll # update the live system as well. my $uri = ""; if ($server_state eq "running") { if ($server_uuid eq $anvil->Get->host_uuid) { ### NOTE: Don't use 'localhost', Sys::Virt translates it to '::1' which Net::OpenSSH ### breaks on. # Connect to the local libvirtd API, and handle problems if not. $uri = "qemu+ssh://".$short_host_name."/system"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uri => $uri }}); } else { my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; my $server_host_name = $anvil->Database->get_host_from_uuid({ short => 1, host_uuid => $server_host_uuid, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_host_name => $server_host_name }}); my $target_ip = $anvil->Network->find_target_ip({ debug => 2, host_uuid => $server_host_uuid, test_access => 1, networks => "bcn,mn,sn,ifn,any", }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { target_ip => $target_ip }}); if ($target_ip) { # Connect using this URI $uri = "qemu+ssh://".$target_ip."/system"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uri => $uri }}); } } } my $connection = ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { uri => $uri }}); eval { $connection = Sys::Virt->new(uri => $uri); }; if ($@) { # Throw an error, then clear the URI so that we just update the DB/on-disk definitions. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, level => 1, key => "warning_0162", variables => { host_name => $short_host_name, uri => $uri, error => $@, }}); return(1); } my $domain = ""; my @domains = $connection->list_all_domains(); foreach my $domain_handle (@domains) { my $this_server_name = $domain_handle->get_name; next if $this_server_name ne $server_name; $domain = $domain_handle; last; } return($domain); } sub manage_boot_menu { my ($anvil) = @_; my $short_host_name = $anvil->Get->short_host_name; my $host_uuid = $anvil->Get->host_uuid; my $server_name = $anvil->data->{switches}{server_name}; my $server_uuid = $anvil->data->{switches}{server_uuid}; my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:short_host_name' => $short_host_name, 's2:host_uuid' => $host_uuid, 's3:server_name' => $server_name, 's4:server_uuid' => $server_uuid, 's5:server_host_uuid' => $server_host_uuid, }}); my $domain_handle = connect_to_libvirt($anvil); my $from_source = get_definition_source($anvil); my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { domain_handle => $domain_handle, from_source => $from_source, server_state => $server_state, }}); if ($server_state eq "running") { my $server_host_name = $anvil->Database->get_host_from_uuid({ short => 1, host_uuid => $server_host_uuid, }); } my $say_yes = $anvil->Words->string({key => 'unit_0001'}); my $say_no = $anvil->Words->string({key => 'unit_0002'}); my $current_boot_menu = lc($anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{info}{boot_menu}); my $say_old_boot_menu = $current_boot_menu eq "yes" ? $say_yes : $say_no; my $new_boot_menu = $anvil->data->{switches}{'boot-menu'}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { current_boot_menu => $current_boot_menu, say_old_boot_menu => $say_old_boot_menu, new_boot_menu => $new_boot_menu, }}); if (($anvil->data->{switches}{'boot-menu'} eq "yes") or ($anvil->data->{switches}{'boot-menu'} eq "no")) { # We need to rewrite the boot menu manually. my $new_definition = ""; my $in_os = 0; my $bootmenu_seen = 0; foreach my $line (split/\n/, $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml}) { $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { line => $line }}); if ($line =~ //) { $in_os = 1; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { in_os => $in_os }}); } if ($in_os) { if ($line =~ //) { my $old_value = $1; $bootmenu_seen = 1; $line =~ s///; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { old_value => $old_value, bootmenu_seen => $bootmenu_seen, line => $line, }}); } if ($line =~ /<\/os>/) { $in_os = 0; if (not $bootmenu_seen) { # Insert it $new_definition .= " \n"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_definition => $new_definition }}); } } } $new_definition .= $line."\n"; } $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { new_definition => $new_definition }}); # Always call this, as a previous run may have only updated some definitions. my $problem = $anvil->Server->update_definition({ debug => 2, server => $server_uuid, new_definition_xml => $new_definition, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { problem => $problem }}); } else { print "Boot Menu enabled: [".$say_old_boot_menu."]\n"; print "- You can change this using '--boot-menu off' or '--boot-menu on'.\n"; } return(0); } sub show_server_details { my ($anvil, $show_nodes) = @_; my $short_host_name = $anvil->Get->short_host_name; my $host_uuid = $anvil->Get->host_uuid; my $server_name = $anvil->data->{switches}{server_name}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:short_host_name' => $short_host_name, 's2:host_uuid' => $host_uuid, 's3:server_name' => $server_name, 's4:show_nodes' => $show_nodes, }}); # Words we'll use. my $say_yes = $anvil->Words->string({key => 'unit_0001'}); my $say_no = $anvil->Words->string({key => 'unit_0002'}); my $say_disk = $anvil->Words->string({key => 'header_0068'}); my $say_optical = $anvil->Words->string({key => 'header_0111'}); my $say_ejected = $anvil->Words->string({key => 'unit_0049'}); my $from_source = get_definition_source($anvil); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }}); # Boot stuff my $say_boot_menu = lc($anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{info}{boot_menu}) eq "yes" ? $say_yes : $say_no; my $say_boot_device = lc($anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{boot_order}{1}{device_type}) eq "cdrom" ? $say_optical : $say_disk; my $boot_device = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{boot_order}{1}{device_target}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { say_boot_menu => $say_boot_menu, say_boot_device => $say_boot_device, boot_device => $boot_device, }}); my $boot_order = []; foreach my $sequence (sort {$a <=> $b} keys %{$anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{boot_order}}) { my $this_device_type = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{boot_order}{$sequence}{device_type}; my $say_this_boot_device = lc($this_device_type) eq "cdrom" ? $say_optical : $say_disk; my $this_boot_device = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{boot_order}{$sequence}{device_target}; my $device_path = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{device}{$this_device_type}{target}{$this_boot_device}{path} // ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { sequence => $sequence, say_this_boot_device => $say_this_boot_device, this_boot_device => $this_boot_device, device_path => $device_path, }}); if ((not $device_path) && (lc($this_device_type) eq "cdrom")) { $device_path = $say_ejected; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { device_path => $device_path }}); } push @{$boot_order}, $sequence."; ".$this_boot_device.", type: [".$say_this_boot_device."], path: [".$device_path."]"; } print "Boot Menu enabled: [".$say_boot_menu."]\n"; print "Boot device:\n"; foreach my $say_other_boot (@{$boot_order}) { print "- ".$say_other_boot."\n"; } # CPU my $cpu_sockets = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{cpu}{sockets}; my $cpu_cores = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{cpu}{cores}; my $cpu_total_cores = $cpu_sockets * $cpu_cores; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { cpu_sockets => $cpu_sockets, cpu_cores => $cpu_cores, cpu_total_cores => $cpu_total_cores, }}); print "CPU Total: [".$cpu_total_cores."]; Sockets: [".$cpu_sockets."], Cores per Socket: [".$cpu_cores."]\n"; # RAM my $ram = $anvil->data->{server}{$short_host_name}{$server_name}{$from_source}{memory}; print "RAM: [".$anvil->Convert->bytes_to_human_readable({'bytes' => $ram})."] (".$anvil->Convert->add_commas({number => $ram})." Bytes)\n"; return(0); } sub get_definition_source { my ($anvil) = @_; my $short_host_name = $anvil->Get->short_host_name; my $server_name = $anvil->data->{switches}{server_name}; my $server_uuid = $anvil->data->{switches}{server_uuid}; my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; my $from_source = ""; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { short_host_name => $short_host_name, server_name => $server_name, server_uuid => $server_uuid, server_state => $server_state, from_source => $from_source, }}); $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh} = "" if not exists $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}; $anvil->data->{server}{$short_host_name}{$server_name}{from_disk} = "" if not exists $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}; $anvil->data->{server}{$short_host_name}{$server_name}{from_db} = "" if not exists $anvil->data->{server}{$short_host_name}{$server_name}{from_db}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "server::${short_host_name}::${server_name}::from_virsh" => $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}, "server::${short_host_name}::${server_name}::from_disk" => $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}, "server::${short_host_name}::${server_name}::from_db" => $anvil->data->{server}{$short_host_name}{$server_name}{from_db}, }}); if (($server_state eq "running") && ($server_host_uuid eq $anvil->Get->host_uuid()) && (exists $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh})) { $from_source = "from_virsh"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }}); } elsif (exists $anvil->data->{server}{$short_host_name}{$server_name}{from_disk}) { $from_source = "from_disk"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }}); } else { $from_source = "from_db"; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { from_source => $from_source }}); } return($from_source); } sub show_server_list { my ($anvil) = @_; # Loop through all Anvil! nodes, then all server in that Anvil! foreach my $anvil_name (sort {$a cmp $b} keys %{$anvil->data->{anvils}{anvil_name}}) { my $anvil_uuid = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_uuid}; my $anvil_description = $anvil->data->{anvils}{anvil_name}{$anvil_name}{anvil_description}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { anvil_name => $anvil_name, anvil_uuid => $anvil_uuid, anvil_description => $anvil_description, }}); if (($anvil->data->{switches}{anvil_uuid}) && ($anvil->data->{switches}{anvil_uuid} ne $anvil_uuid)) { next; } print "\n".$anvil->Words->string({key => "message_0343", variables => { anvil_name => $anvil_name, anvil_uuid => $anvil_uuid, anvil_description => $anvil_description, }})."\n"; my $server_count = 0; if (exists $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}) { $server_count = keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_count => $server_count }}); } if (not $server_count) { print $anvil->Words->string({key => "message_0344"})."\n"; } else { foreach my $server_name (sort {$a cmp $b} keys %{$anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}}) { my $server_uuid = $anvil->data->{servers}{anvil_uuid}{$anvil_uuid}{server_name}{$server_name}{server_uuid}; my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { server_name => $server_name, server_uuid => $server_uuid, server_state => $server_state, }}); next if $server_state eq "DELETED"; print $anvil->Words->string({key => "message_0345", variables => { server_name => $server_name, server_uuid => $server_uuid, }})."\n"; } } } return(0); } sub validate_server { my ($anvil) = @_; $anvil->Get->server_from_switch({ debug => 2, string => $anvil->data->{switches}{server}, anvil_uuid => $anvil->data->{switches}{anvil_uuid}, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::server_name" => $anvil->data->{switches}{server_name}, "switches::server_uuid" => $anvil->data->{switches}{server_uuid}, }}); if (not $anvil->data->{switches}{server_uuid}) { show_server_list($anvil); my $variables = { server => $anvil->data->{switches}{server}, anvil => $anvil->data->{switches}{anvil_name}, }; if ($anvil->data->{switches}{anvil_uuid}) { # Not found on the requested Anvil! node. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0451", variables => $variables}); $anvil->Job->update_progress({ progress => 100, message => "error_0451", variables => $variables, }) if $anvil->data->{switches}{'job-uuid'}; } else { # Not found at all. $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0452", variables => $variables}); $anvil->Job->update_progress({ progress => 100, message => "error_0452", variables => $variables, }) if $anvil->data->{switches}{'job-uuid'}; } $anvil->nice_exit({exit_code => 1}); } my $variables = { server_name => $anvil->data->{switches}{server_name}, server_uuid => $anvil->data->{switches}{server_uuid}, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 1, key => "log_0802", variables => $variables}); $anvil->Job->update_progress({ progress => 2, message => "log_0802", variables => $variables, }) if $anvil->data->{switches}{'job-uuid'}; my $short_host_name = $anvil->Get->short_host_name; my $server_name = $anvil->data->{switches}{server_name}; my $server_uuid = $anvil->data->{switches}{server_uuid}; my $server_definition = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_definition_xml}; my $server_host_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_host_uuid}; my $server_state = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_state}; my $anvil_uuid = $anvil->data->{servers}{server_uuid}{$server_uuid}{server_anvil_uuid}; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:short_host_name' => $short_host_name, 's2:server_name' => $server_name, 's3:server_uuid' => $server_uuid, 's4:server_definition' => $server_definition, 's5:server_host_uuid' => $server_host_uuid, 's6:server_state' => $server_state, 's7:anvil_uuid' => $anvil_uuid, }}); if (not $anvil->data->{switches}{anvil_uuid}) { $anvil->data->{switches}{anvil_uuid} = $anvil_uuid; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:switches::anvil_uuid' => $anvil->data->{switches}{anvil_uuid}, }}); } if ($server_state eq "DELETED") { # The server has been deleted my $variables = { server => $server_name }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0453", variables => $variables}); $anvil->Job->update_progress({ progress => 100, message => "error_0453", variables => $variables, }) if $anvil->data->{switches}{'job-uuid'}; $anvil->nice_exit({exit_code => 1}); } # Parse the definition. $anvil->Server->parse_definition({ debug => 3, host => $short_host_name, server => $server_name, source => "from_db", definition => $server_definition, }); # Can we read the XML definition? $anvil->Server->get_status({ debug => 2, server => $server_name, host => $short_host_name, }); if (not $anvil->data->{server}{$short_host_name}{$server_name}{from_virsh}{xml}) { # The server isn't actually running... Not here anyway. if ($server_state eq "running") { my $server_host_name = $anvil->Database->get_host_from_uuid({ short => 1, host_uuid => $server_host_uuid, }); my $variables = { server => $server_name, host_name => $server_host_name, }; $anvil->Log->entry({source => $THIS_FILE, line => __LINE__, 'print' => 1, level => 0, priority => "err", key => "error_0454", variables => $variables}); $anvil->Job->update_progress({ progress => 100, message => "error_0454", variables => $variables, }) if $anvil->data->{switches}{'job-uuid'}; $anvil->nice_exit({exit_code => 1}); } } return(0); } sub load_job { my ($anvil) = @_; $anvil->Job->clear({ debug => 2, job_uuid => $anvil->data->{switches}{'job-uuid'}, }); $anvil->Job->get_job_details({ debug => 2, job_uuid => $anvil->data->{switches}{'job-uuid'}, }); $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 'jobs::job_data' => $anvil->data->{jobs}{job_data}, }}); foreach my $line (split/\n/, $anvil->data->{jobs}{job_data}) { my ($variable, $value) = ($line =~ /^(.*)=(.*)$/); $value =~ s/^"(.*)\"/$1/; $value =~ s/^'(.*)\'/$1/; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { 's1:line' => $line, 's2:variable' => $variable, 's3:value' => $value, }}); $anvil->data->{switches}{$variable} = $value; $anvil->Log->variables({source => $THIS_FILE, line => __LINE__, level => 2, list => { "switches::${variable}" => $anvil->data->{switches}{$variable}, }}); } $anvil->Job->update_progress({ progress => 1, job_picked_up_by => $$, job_picked_up_at => time, message => "message_0350", }); return(0); }