HTTP File Downloader (cpp, written by Codex)

Codex wrote this cpp project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTTP File Downloader

Write a program that downloads files from HTTP/HTTPS URLs with support for resumable downloads, progress tracking, concurrent downloads, and checksum verification.

FUNCTIONAL REQUIREMENTS:
- Accept one or more URLs as command-line arguments
- Display a progress bar during download showing: percentage complete, bytes downloaded, total size, download speed, and estimated time remaining
- Support resumable downloads via HTTP Range headers: if a download is interrupted, restarting with the same URL and output path should resume from where it stopped via --resume flag
- Support concurrent downloading of multiple files via --parallel flag with configurable thread count (--threads, default 4)
- Support downloading all URLs listed in a text file (one URL per line) via --list flag
- Verify downloaded file integrity via --checksum flag accepting algorithm:hash format (e.g., --checksum sha256:abc123...)
- Support custom HTTP headers via --header flag (e.g., --header "Authorization: Bearer token")
- Support following HTTP redirects (up to 10 hops) and report the final URL
- Set connection timeout via --timeout flag (default 30 seconds) and retry failed downloads via --retries flag (default 3) with exponential backoff
- Save files to a directory specified by --output flag (default: current directory), using the filename from the URL or Content-Disposition header
- Print a download summary to console: file name, size, time taken, average speed, and checksum verification result
- If no URLs are given, download a set of sample public domain text files from Project Gutenberg, display progress for each, and print a summary table
- Handle errors: DNS resolution failures, SSL certificate errors, HTTP 4xx/5xx responses, disk full, and network timeouts

Create a complete C++ project for a clean Ubuntu 22.04 machine with only G++ 12+ and CMake 3.22+ installed. Include:
- Source code
- CMakeLists.txt with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

cmake_minimum_required(VERSION 3.22)
project(http_file_downloader_cpp VERSION 1.0.0 LANGUAGES C CXX)

set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

include(FetchContent)

FetchContent_Declare(
  openssl
  URL https://github.com/openssl/openssl/releases/download/openssl-3.4.0/openssl-3.4.0.tar.gz
)
FetchContent_MakeAvailable(openssl)

FetchContent_Declare(
  curl
  URL https://github.com/curl/curl/releases/download/curl-8_10_1/curl-8.10.1.tar.xz
)
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
set(BUILD_CURL_EXE OFF CACHE BOOL "" FORCE)
set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
set(CURL_USE_OPENSSL ON CACHE BOOL "" FORCE)
FetchContent_MakeAvailable(curl)

add_executable(http_file_downloader src/main.cpp)
target_link_libraries(http_file_downloader PRIVATE CURL::libcurl OpenSSL::Crypto OpenSSL::SSL)

# HTTP File Downloader (C++)

Downloads HTTP/HTTPS files with resume support (`Range`), progress tracking, retries with exponential backoff, redirect following, custom headers, optional parallel workers, checksum verification, and summary reporting.

## Requirements

- Ubuntu 22.04
- G++ 12+
- CMake 3.22+

## Dependencies (Pinned)

- OpenSSL `3.4.0`
- cURL `8.10.1`

Dependencies are pinned in `CMakeLists.txt` via `FetchContent`.

## Build

```bash
cmake -S . -B build
cmake --build build -j
```

## Run

```bash
./build/http_file_downloader https://example.com/file.txt
./build/http_file_downloader https://example.com/file.bin --resume --output ./downloads
./build/http_file_downloader --list urls.txt --parallel --threads 4 --timeout 30 --retries 3
./build/http_file_downloader https://example.com/file --checksum sha256:abc123...
./build/http_file_downloader https://example.com/file --header "Authorization: Bearer TOKEN"
./build/http_file_downloader
```

## Notes

- `--parallel` enables concurrent downloads (`--threads` default `4`).
- `--resume` continues from local file size when server supports range requests.
- Redirects are followed up to 10 hops and final URL is reported.
- No-input mode downloads sample Project Gutenberg text files.

#include <curl/curl.h>
#include <openssl/evp.h>

#include <chrono>
#include <cmath>
#include <filesystem>
#include <fstream>
#include <future>
#include <iomanip>
#include <iostream>
#include <map>
#include <mutex>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>

namespace fs = std::filesystem;

struct CliOptions {
    std::map<std::string, std::string> kv;
    std::vector<std::string> headers;
    std::vector<std::string> positional;
};

struct Checksum {
    std::string algorithm;
    std::string expected;
};

struct DownloadResult {
    std::string url;
    std::string finalUrl;
    std::string file;
    std::uint64_t size{};
    double timeSec{};
    double avgSpeed{};
    std::string checksum;
};

struct ProgressState {
    std::string fileName;
    int index{};
    int total{};
    std::uint64_t initialOffset{};
    std::chrono::steady_clock::time_point start;
    std::mutex* outputMutex{};
};

struct HeaderState {
    std::string contentDisposition;
};

const std::vector<std::string> SAMPLE_URLS = {
    "https://www.gutenberg.org/files/11/11-0.txt",
    "https://www.gutenberg.org/files/1342/1342-0.txt",
    "https://www.gutenberg.org/files/1661/1661-0.txt"
};

CliOptions parseArgs(int argc, char** argv) {
    CliOptions out;
    for (int i = 1; i < argc; ++i) {
        std::string token = argv[i];
        if (token.rfind("--", 0) == 0) {
            std::string key = token.substr(2);
            std::string value = "true";
            if (i + 1 < argc) {
                std::string next = argv[i + 1];
                if (next.rfind("--", 0) != 0) {
                    value = next;
                    ++i;
                }
            }
            if (key == "header") out.headers.push_back(value);
            else out.kv[key] = value;
        } else out.positional.push_back(token);
    }
    return out;
}

std::optional<Checksum> parseChecksum(const std::string& raw) {
    if (raw.empty()) return std::nullopt;
    auto pos = raw.find(':');
    if (pos == std::string::npos || pos == 0 || pos + 1 >= raw.size()) {
        throw std::invalid_argument("Invalid --checksum. Use algorithm:hash.");
    }
    Checksum c;
    c.algorithm = raw.substr(0, pos);
    for (char& ch : c.algorithm) ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
    c.expected = raw.substr(pos + 1);
    for (char& ch : c.expected) ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
    return c;
}

std::string filenameFromUrl(const std::string& url) {
    auto slash = url.find_last_of('/');
    if (slash == std::string::npos || slash + 1 >= url.size()) return "download.bin";
    std::string name = url.substr(slash + 1);
    auto q = name.find('?');
    if (q != std::string::npos) name = name.substr(0, q);
    return name.empty() ? "download.bin" : name;
}

std::string trim(std::string s) {
    while (!s.empty() && std::isspace(static_cast<unsigned char>(s.front()))) s.erase(s.begin());
    while (!s.empty() && std::isspace(static_cast<unsigned char>(s.back()))) s.pop_back();
    return s;
}

std::string filenameFromContentDisposition(const std::string& cd) {
    auto pos = cd.find("filename=");
    if (pos == std::string::npos) return "";
    std::string value = cd.substr(pos + 9);
    value = trim(value);
    if (!value.empty() && value.front() == '"') value.erase(value.begin());
    if (!value.empty() && value.back() == '"') value.pop_back();
    auto semicolon = value.find(';');
    if (semicolon != std::string::npos) value = value.substr(0, semicolon);
    return trim(value);
}

std::string formatBytes(double n) {
    if (!std::isfinite(n) || n < 0) return "?";
    const char* units[] = {"B", "KB", "MB", "GB", "TB"};
    int i = 0;
    while (n >= 1024.0 && i < 4) {
        n /= 1024.0;
        ++i;
    }
    std::ostringstream oss;
    if (i == 0) oss << std::fixed << std::setprecision(0);
    else oss << std::fixed << std::setprecision(2);
    oss << n << " " << units[i];
    return oss.str();
}

std::string formatEta(double seconds) {
    if (!std::isfinite(seconds) || seconds < 0) return "--:--";
    int s = static_cast<int>(std::floor(seconds)) % 60;
    int m = static_cast<int>(std::floor(seconds / 60.0)) % 60;
    int h = static_cast<int>(std::floor(seconds / 3600.0));
    std::ostringstream oss;
    if (h > 0) oss << h << ":" << std::setw(2) << std::setfill('0') << m << ":" << std::setw(2) << s;
    else oss << std::setw(2) << std::setfill('0') << m << ":" << std::setw(2) << s;
    return oss.str();
}

size_t writeCallback(char* ptr, size_t size, size_t nmemb, void* userdata) {
    auto* out = static_cast<std::ofstream*>(userdata);
    size_t total = size * nmemb;
    out->write(ptr, static_cast<std::streamsize>(total));
    if (!*out) return 0;
    return total;
}

size_t headerCallback(char* buffer, size_t size, size_t nitems, void* userdata) {
    size_t total = size * nitems;
    auto* hs = static_cast<HeaderState*>(userdata);
    std::string line(buffer, total);
    std::string lower = line;
    for (char& c : lower) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
    if (lower.rfind("content-disposition:", 0) == 0) {
        hs->contentDisposition = trim(line.substr(std::string("content-disposition:").size()));
    }
    return total;
}

int progressCallback(void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) {
    auto* ps = static_cast<ProgressState*>(clientp);
    double total = dltotal > 0 ? static_cast<double>(dltotal + static_cast<curl_off_t>(ps->initialOffset)) : 0.0;
    double downloaded = static_cast<double>(dlnow + static_cast<curl_off_t>(ps->initialOffset));
    double elapsed = std::max(0.001, std::chrono::duration<double>(std::chrono::steady_clock::now() - ps->start).count());
    double speed = (downloaded - static_cast<double>(ps->initialOffset)) / elapsed;
    double pct = total > 0 ? downloaded * 100.0 / total : 0.0;
    double eta = (speed > 0.0 && total > 0.0) ? (total - downloaded) / speed : std::numeric_limits<double>::infinity();

    std::lock_guard<std::mutex> lk(*ps->outputMutex);
    std::cout << "\r[" << (ps->index + 1) << "/" << ps->total << "] " << ps->fileName
              << " | " << std::fixed << std::setprecision(1) << pct << "% | "
              << formatBytes(downloaded) << "/" << (total > 0 ? formatBytes(total) : "?")
              << " | " << formatBytes(speed) << "/s | ETA " << formatEta(eta) << "   " << std::flush;
    return 0;
}

std::string hashFile(const fs::path& file, const std::string& algorithm) {
    const EVP_MD* md = EVP_get_digestbyname(algorithm.c_str());
    if (!md) throw std::runtime_error("Unsupported checksum algorithm: " + algorithm);
    EVP_MD_CTX* ctx = EVP_MD_CTX_new();
    if (!ctx) throw std::runtime_error("Failed to create digest context");
    if (EVP_DigestInit_ex(ctx, md, nullptr) != 1) {
        EVP_MD_CTX_free(ctx);
        throw std::runtime_error("Digest init failed");
    }
    std::ifstream in(file, std::ios::binary);
    std::vector<char> buf(1024 * 1024);
    while (in) {
        in.read(buf.data(), static_cast<std::streamsize>(buf.size()));
        auto got = in.gcount();
        if (got > 0) {
            if (EVP_DigestUpdate(ctx, buf.data(), static_cast<size_t>(got)) != 1) {
                EVP_MD_CTX_free(ctx);
                throw std::runtime_error("Digest update failed");
            }
        }
    }
    unsigned char out[EVP_MAX_MD_SIZE];
    unsigned int outLen = 0;
    if (EVP_DigestFinal_ex(ctx, out, &outLen) != 1) {
        EVP_MD_CTX_free(ctx);
        throw std::runtime_error("Digest finalize failed");
    }
    EVP_MD_CTX_free(ctx);
    std::ostringstream oss;
    for (unsigned int i = 0; i < outLen; ++i) oss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(out[i]);
    return oss.str();
}

DownloadResult downloadOne(
    const std::string& url,
    int index,
    int totalCount,
    const CliOptions& opts,
    const std::optional<Checksum>& checksum,
    std::mutex& outputMutex
) {
    const int timeout = std::max(1, std::stoi(opts.kv.count("timeout") ? opts.kv.at("timeout") : "30"));
    const int retries = std::max(0, std::stoi(opts.kv.count("retries") ? opts.kv.at("retries") : "3"));
    const bool resume = opts.kv.count("resume") > 0;
    const fs::path outDir = fs::absolute(opts.kv.count("output") ? opts.kv.at("output") : ".");
    fs::create_directories(outDir);

    std::string guessedName = filenameFromUrl(url);
    fs::path guessedPath = outDir / guessedName;
    std::uint64_t existing = (resume && fs::exists(guessedPath)) ? fs::file_size(guessedPath) : 0;

    for (int attempt = 0; attempt <= retries; ++attempt) {
        auto started = std::chrono::steady_clock::now();
        CURL* curl = curl_easy_init();
        if (!curl) throw std::runtime_error("Failed to initialize curl");

        std::ofstream out;
        HeaderState hs{};
        ProgressState ps{};
        ps.fileName = guessedName;
        ps.index = index;
        ps.total = totalCount;
        ps.initialOffset = existing;
        ps.start = started;
        ps.outputMutex = &outputMutex;

        struct curl_slist* headerList = nullptr;
        for (const auto& h : opts.headers) headerList = curl_slist_append(headerList, h.c_str());

        std::string finalUrl;
        long responseCode = 0;
        try {
            out.open(guessedPath, (existing > 0 ? std::ios::app : std::ios::trunc) | std::ios::binary);
            if (!out) throw std::runtime_error("Cannot open output file (disk full/read-only?): " + guessedPath.string());

            curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
            curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
            curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10L);
            curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout);
            curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout * 10L);
            curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0L);
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
            curl_easy_setopt(curl, CURLOPT_HEADERDATA, &hs);
            curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback);
            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
            curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progressCallback);
            curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &ps);
            curl_easy_setopt(curl, CURLOPT_USERAGENT, "tmlr-http-downloader/1.0");
            curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
            curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L);
            if (headerList) curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerList);
            if (existing > 0) curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, static_cast<curl_off_t>(existing));

            CURLcode rc = curl_easy_perform(curl);
            curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
            char* effective = nullptr;
            curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective);
            finalUrl = effective ? std::string(effective) : url;
            out.close();
            {
                std::lock_guard<std::mutex> lk(outputMutex);
                std::cout << "\n";
            }

            if (rc != CURLE_OK) throw std::runtime_error(std::string("curl error: ") + curl_easy_strerror(rc));
            if (responseCode >= 400) throw std::runtime_error("HTTP " + std::to_string(responseCode));

            std::string cdName = filenameFromContentDisposition(hs.contentDisposition);
            fs::path finalPath = guessedPath;
            if (!cdName.empty() && (existing == 0 || !resume)) {
                fs::path target = outDir / cdName;
                if (target != guessedPath) {
                    fs::rename(guessedPath, target);
                    finalPath = target;
                    guessedName = cdName;
                }
            }

            std::uint64_t size = fs::file_size(finalPath);
            double sec = std::max(0.001, std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count());
            std::string checksumResult = "not requested";
            if (checksum) {
                std::string got = hashFile(finalPath, checksum->algorithm);
                checksumResult = (got == checksum->expected) ? "ok" : ("mismatch (" + got + ")");
            }

            if (headerList) curl_slist_free_all(headerList);
            curl_easy_cleanup(curl);
            return {url, finalUrl, finalPath.filename().string(), size, sec, size / sec, checksumResult};
        } catch (const std::exception&) {
            if (headerList) curl_slist_free_all(headerList);
            curl_easy_cleanup(curl);
            if (attempt >= retries) throw;
            std::this_thread::sleep_for(std::chrono::seconds(1 << attempt));
        }
    }
    throw std::runtime_error("Failed after retries");
}

void printSummary(const std::vector<DownloadResult>& rows) {
    std::cout << "\nDownload Summary\n";
    std::cout << "File | Size | Time | Avg Speed | Checksum | Final URL\n";
    std::cout << "---- | ---- | ---- | --------- | -------- | ---------\n";
    for (const auto& r : rows) {
        std::cout << r.file << " | " << formatBytes(static_cast<double>(r.size)) << " | "
                  << std::fixed << std::setprecision(2) << r.timeSec << "s | "
                  << formatBytes(r.avgSpeed) << "/s | " << r.checksum << " | " << r.finalUrl << "\n";
    }
}

int main(int argc, char** argv) {
    try {
        CliOptions opts = parseArgs(argc, argv);
        std::vector<std::string> urls = opts.positional;
        if (opts.kv.count("list")) {
            fs::path listFile = fs::absolute(opts.kv.at("list"));
            std::ifstream in(listFile);
            if (!in) throw std::runtime_error("Cannot read list file: " + listFile.string());
            std::string line;
            while (std::getline(in, line)) {
                line = trim(line);
                if (!line.empty()) urls.push_back(line);
            }
        }
        if (urls.empty()) urls = SAMPLE_URLS;

        auto checksum = parseChecksum(opts.kv.count("checksum") ? opts.kv.at("checksum") : "");
        bool parallel = opts.kv.count("parallel") > 0;
        int threads = std::max(1, std::stoi(opts.kv.count("threads") ? opts.kv.at("threads") : "4"));

        if (curl_global_init(CURL_GLOBAL_DEFAULT) != 0) throw std::runtime_error("curl_global_init failed");
        std::mutex outputMutex;
        std::vector<DownloadResult> results;

        if (parallel && urls.size() > 1) {
            std::vector<std::future<DownloadResult>> futures;
            std::vector<std::thread> workers;
            std::mutex queueMutex;
            size_t nextIdx = 0;
            std::vector<std::optional<DownloadResult>> ordered(urls.size());
            std::vector<std::string> errors(urls.size());

            auto worker = [&]() {
                while (true) {
                    size_t idx = 0;
                    {
                        std::lock_guard<std::mutex> lk(queueMutex);
                        if (nextIdx >= urls.size()) break;
                        idx = nextIdx++;
                    }
                    try {
                        auto ck = (idx == 0) ? checksum : std::nullopt;
                        ordered[idx] = downloadOne(urls[idx], static_cast<int>(idx), static_cast<int>(urls.size()), opts, ck, outputMutex);
                    } catch (const std::exception& ex) {
                        errors[idx] = ex.what();
                    }
                }
            };

            for (int t = 0; t < threads; ++t) workers.emplace_back(worker);
            for (auto& w : workers) w.join();

            for (size_t i = 0; i < urls.size(); ++i) {
                if (ordered[i]) results.push_back(*ordered[i]);
                else results.push_back({urls[i], "n/a", "n/a", 0, 0, 0, "failed: " + errors[i]});
            }
        } else {
            for (size_t i = 0; i < urls.size(); ++i) {
                auto ck = (i == 0) ? checksum : std::nullopt;
                results.push_back(downloadOne(urls[i], static_cast<int>(i), static_cast<int>(urls.size()), opts, ck, outputMutex));
            }
        }

        printSummary(results);
        curl_global_cleanup();
        return 0;
    } catch (const std::exception& ex) {
        std::cerr << "Error: " << ex.what() << "\n";
        return 1;
    }
}