Skip to content
ConsoleMonster

Backup jobs with retry

Run a bounded wave, retain its outcomes and retry only unfinished jobs.

Demos Dashboards

Use it when independent backup jobs need a concurrency limit and a bounded recovery attempt.

  • Live
  • 100×22
  • PowerShell 7.4+
  • .NET 8.0+
Backup jobs with retry 100×2224-bitFull

How it's built

The whole program in one file, with its sample data at the top. It drew the recording above; both languages draw the same cells.

PowerShellbackup-runner.ps1View on GitHub
# Run a bounded backup wave and retry only jobs that did not return a result.
param([switch] $StaticSnapshot, [ValidateRange(0, 2)][int] $Step = 0)
Import-Module ConsoleMonster

# Data: synthetic backup jobs; no files, databases or remote services are touched.
$jobs = @(
    [pscustomobject] @{ Name = 'catalog'; Size = '2.4 GiB'; Retry = $false }
    [pscustomobject] @{ Name = 'media'; Size = '8.1 GiB'; Retry = $true }
    [pscustomobject] @{ Name = 'settings'; Size = '14 MiB'; Retry = $false }
    [pscustomobject] @{ Name = 'audit'; Size = '640 MiB'; Retry = $false }
)

# Layout: queued, first-wave and recovered states use the same result columns.
function New-BackupFrame($Tick) {
    $rows = foreach ($job in $jobs) {
        $status = if ($Tick -eq 0) { 'Pending' } elseif ($Tick -eq 1 -and $job.Retry) { 'Warn' } else { 'Pass' }
        $attempts = if ($Tick -eq 0) { 0 } elseif ($Tick -eq 2 -and $job.Retry) { 2 } else { 1 }
        [pscustomobject] @{ Job = $job.Name; Size = $job.Size; Attempts = $attempts; Status = $status }
    }
    New-ConsoleRows -Gap 1 {
        New-ConsoleRule '[bold]Nightly backup[/] [muted]· throttle 2 · maximum 2 attempts[/]'
        New-ConsoleTable -InputObject $rows -Property Job, Size, Attempts, Status -Expand
        if ($Tick -eq 0) { '[muted]4 jobs queued · destination ./backups/nightly[/]' }
        elseif ($Tick -eq 1) { '[pass]3 passed[/]   [warn]1 queued for retry[/]' }
        else { '[pass]4 passed[/]   [accent]1 recovered on retry[/]   [muted]retention: 14 days[/]' }
    }
}

# Behaviour: headless snapshots do not start workers. Live jobs emit their result on success.
if ($StaticSnapshot -or [Console]::IsInputRedirected) {
    New-BackupFrame $Step | Write-ConsoleWidget
    return
}
$pending = $jobs
$completed = @()
$attemptErrors = @()
foreach ($attempt in 1..2) {
    if ($pending.Count -eq 0) { break }
    $results = @($pending | ForEach-ConsoleItem -Parallel -ThrottleLimit 2 -Name Name -Title "Backup attempt $attempt" -ErrorAction SilentlyContinue -ErrorVariable failures {
        Start-Sleep -Milliseconds 250
        if ($_.Retry -and $using:attempt -eq 1) { throw 'Destination busy; retry this job' }
        [pscustomobject] @{ Name = $_.Name; Archive = "./backups/nightly/$($_.Name).zip" }
    })
    $completed += $results
    $attemptErrors += $failures
    $pending = @($pending | Where-Object { $_.Name -notin $results.Name })
}
# $completed holds archive results; $pending and $attemptErrors retain unresolved work and diagnostics.
New-BackupFrame 2 | Write-ConsoleWidget
C#Program.csView on GitHub
// Run a bounded backup wave and retry only jobs that did not return a result.
using ConsoleMonster;
using ConsoleMonster.Layout;
using ConsoleMonster.Live;
using ConsoleMonster.Presenters;
using ConsoleMonster.Widgets;

// Data: synthetic backup jobs; no files, databases or remote services are touched.
var jobs = new[] {
    new { Name = "catalog", Size = "2.4 GiB", Retry = false },
    new { Name = "media", Size = "8.1 GiB", Retry = true },
    new { Name = "settings", Size = "14 MiB", Retry = false },
    new { Name = "audit", Size = "640 MiB", Retry = false }
};

// Layout: queued, first-wave and recovered states use the same result columns.
IWidget Frame(int tick) {
    Table table = new Table { Expand = true };
    table.AddColumn("Job").AddColumn("Size").AddColumn("Attempts")
        .AddColumn("Status", column => column.Presenter = CellPresenters.StatusWord);
    foreach (var job in jobs) {
        string status = tick == 0 ? "Pending" : tick == 1 && job.Retry ? "Warn" : "Pass";
        int attempts = tick == 0 ? 0 : tick == 2 && job.Retry ? 2 : 1;
        table.AddRow(job.Name, job.Size, TableCell.FromValue(attempts), TableCell.FromValue(status));
    }
    string summary = tick == 0 ? "[muted]4 jobs queued · destination ./backups/nightly[/]"
        : tick == 1 ? "[pass]3 passed[/]   [warn]1 queued for retry[/]"
        : "[pass]4 passed[/]   [accent]1 recovered on retry[/]   [muted]retention: 14 days[/]";
    return new Rows(new Rule("[bold]Nightly backup[/] [muted]· throttle 2 · maximum 2 attempts[/]"),
        table, new TextBlock(summary)) { Gap = 1 };
}

// Behaviour: headless snapshots do not start workers. Live jobs emit their result on success.
int at = Array.IndexOf(args, "--step");
int step = at < 0 ? 0 : int.Parse(args[at + 1], System.Globalization.CultureInfo.InvariantCulture);
if (step < 0 || step > 2) throw new ArgumentOutOfRangeException(nameof(step));
if (args.Contains("--snapshot") || (!args.Contains("--interactive") && Console.IsInputRedirected)) {
    Terminal.Write(Frame(step));
    return;
}
var pending = jobs;
List<string> completed = new List<string>();
List<Exception> attemptErrors = new List<Exception>();
for (int attempt = 1; attempt <= 2 && pending.Length > 0; attempt++) {
    var outcomes = await ConsoleRun.ForEachAsync(pending, async (job, cancel) => {
        await Task.Delay(250, cancel);
        if (job.Retry && attempt == 1) throw new InvalidOperationException("Destination busy; retry this job");
        return "./backups/nightly/" + job.Name + ".zip";
    }, new RunOptions { ThrottleLimit = 2, Title = "Backup attempt " + attempt });
    completed.AddRange(outcomes.Where(result => result.Succeeded).Select(result => result.Value));
    attemptErrors.AddRange(outcomes.Where(result => !result.Succeeded).Select(result => result.Error!));
    pending = outcomes.Where(result => !result.Succeeded).Select(result => result.Item).ToArray();
}
// completed holds archive results; pending and attemptErrors retain unresolved work and diagnostics.
Terminal.Write(Frame(2));

Run it yourself

Three steps in PowerShell, four in C#. Every command copies with one click.

  1. Install the module

    Once. Windows PowerShell 5.1 or PowerShell 7 on Windows, macOS or Linux.

    Install-Module ConsoleMonster -Scope CurrentUser

  2. Save the program

    Download it, or copy it from above into backup-runner.ps1.

    Download backup-runner.ps1View on GitHub

  3. Run it

    From the folder you saved it in.

    ./backup-runner.ps1

  1. Create a project

    A plain console app; .NET 8 or later.

    dotnet new console -n Backuprunner; cd Backuprunner

  2. Add ConsoleMonster

    The only package the program needs.

    dotnet add package ConsoleMonster

  3. Replace Program.cs

    Download it, or copy the program from above.

    Download Program.csView on GitHub

  4. Run it

    dotnet run

Or skip the copying. Every demo ships with the module and the project template.

Show-ConsoleDemo backup-runner
Get-ConsoleDemo backup-runner -Path ./backup-runner.ps1
dotnet new install ConsoleMonster.Templates
dotnet new consolemonster --demo backup-runner -n Backuprunner

Make it yours

Start with these, then read the rest of the file.

  1. Replace the synthetic delay with an operation that returns an archive result only on success.

  2. Set the concurrency limit and maximum attempts for your storage system.

  3. Persist completed archives and retained errors before deciding how to handle unresolved jobs.

More dashboards

All demos