HTTP File Downloader (javascript, written by Codex)

Codex wrote this javascript project from the task below. It installed and ran on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTTP File Downloader

Write a program that downloads files from HTTP/HTTPS URLs with support for resumable downloads, progress tracking, concurrent downloads, and checksum verification.

FUNCTIONAL REQUIREMENTS:
- Accept one or more URLs as command-line arguments
- Display a progress bar during download showing: percentage complete, bytes downloaded, total size, download speed, and estimated time remaining
- Support resumable downloads via HTTP Range headers: if a download is interrupted, restarting with the same URL and output path should resume from where it stopped via --resume flag
- Support concurrent downloading of multiple files via --parallel flag with configurable thread count (--threads, default 4)
- Support downloading all URLs listed in a text file (one URL per line) via --list flag
- Verify downloaded file integrity via --checksum flag accepting algorithm:hash format (e.g., --checksum sha256:abc123...)
- Support custom HTTP headers via --header flag (e.g., --header "Authorization: Bearer token")
- Support following HTTP redirects (up to 10 hops) and report the final URL
- Set connection timeout via --timeout flag (default 30 seconds) and retry failed downloads via --retries flag (default 3) with exponential backoff
- Save files to a directory specified by --output flag (default: current directory), using the filename from the URL or Content-Disposition header
- Print a download summary to console: file name, size, time taken, average speed, and checksum verification result
- If no URLs are given, download a set of sample public domain text files from Project Gutenberg, display progress for each, and print a summary table
- Handle errors: DNS resolution failures, SSL certificate errors, HTTP 4xx/5xx responses, disk full, and network timeouts

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

{
  "name": "http-file-downloader",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "http-file-downloader",
      "version": "1.0.0",
      "dependencies": {
        "got": "14.4.3",
        "p-limit": "5.0.0"
      },
      "engines": {
        "node": ">=20.0.0"
      }
    },
    "node_modules/got": {
      "version": "14.4.3"
    },
    "node_modules/p-limit": {
      "version": "5.0.0"
    }
  }
}

{
  "name": "http-file-downloader",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "main": "src/index.js",
  "scripts": {
    "start": "node src/index.js"
  },
  "engines": {
    "node": ">=20.0.0"
  },
  "dependencies": {
    "got": "14.4.3",
    "p-limit": "5.0.0"
  }
}

# HTTP File Downloader (JavaScript)

Downloads files from HTTP/HTTPS with progress display, resumable downloads, retry/backoff, redirect handling, parallel mode, checksum verification, custom headers, list-file input, and batch summary output.

## Requirements

- Ubuntu 22.04
- Node.js 20+ (LTS)

## Dependencies (Pinned)

- `got@14.4.3`
- `p-limit@5.0.0`

## Setup

```bash
npm install
```

## Run

```bash
node src/index.js https://example.com/file.txt
node src/index.js https://example.com/file.bin --resume --output ./downloads
node src/index.js --list urls.txt --parallel --threads 4 --timeout 30 --retries 3
node src/index.js https://example.com/file --checksum sha256:abc123...
node src/index.js https://example.com/file --header "Authorization: Bearer TOKEN"
node src/index.js
```

## Notes

- `--parallel` enables concurrent downloads, bounded by `--threads` (default `4`).
- `--resume` uses HTTP `Range` where supported.
- `--checksum` applies verification to the first item in a multi-download run.
- No-input mode downloads sample Project Gutenberg text files.

import fs from "fs";
import path from "path";
import crypto from "crypto";
import { URL } from "url";
import got from "got";
import pLimit from "p-limit";

const SAMPLE_URLS = [
  "https://www.gutenberg.org/files/11/11-0.txt",
  "https://www.gutenberg.org/files/1342/1342-0.txt",
  "https://www.gutenberg.org/files/1661/1661-0.txt",
];

function parseArgs(argv) {
  const options = {};
  const positional = [];
  for (let i = 0; i < argv.length; i += 1) {
    const t = argv[i];
    if (t.startsWith("--")) {
      const key = t.slice(2);
      const next = argv[i + 1];
      const value = next && !next.startsWith("--") ? next : true;
      if (value !== true) i += 1;
      if (key === "header") {
        if (!options.header) options.header = [];
        options.header.push(String(value));
      } else options[key] = value;
    } else positional.push(t);
  }
  return { options, positional };
}

function parseHeaders(values = []) {
  const out = {};
  for (const h of values) {
    const idx = h.indexOf(":");
    if (idx <= 0) continue;
    const k = h.slice(0, idx).trim();
    const v = h.slice(idx + 1).trim();
    if (k) out[k] = v;
  }
  return out;
}

function parseChecksum(raw) {
  if (!raw) return null;
  const idx = raw.indexOf(":");
  if (idx <= 0) throw new Error("Invalid --checksum. Use algorithm:hash");
  const algorithm = raw.slice(0, idx).toLowerCase();
  const expected = raw.slice(idx + 1).toLowerCase();
  return { algorithm, expected };
}

function fromContentDisposition(v) {
  if (!v) return null;
  const m = /filename\*?=(?:UTF-8'')?"?([^\";]+)"?/i.exec(v);
  return m ? decodeURIComponent(m[1]) : null;
}

function filenameFromUrl(url) {
  try {
    const u = new URL(url);
    const base = path.basename(u.pathname || "");
    return base && base !== "/" ? base : "download.bin";
  } catch {
    return "download.bin";
  }
}

function formatBytes(n) {
  if (!Number.isFinite(n) || n < 0) return "?";
  const units = ["B", "KB", "MB", "GB", "TB"];
  let v = n;
  let i = 0;
  while (v >= 1024 && i < units.length - 1) {
    v /= 1024;
    i += 1;
  }
  return `${v.toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
}

function formatEta(sec) {
  if (!Number.isFinite(sec) || sec < 0) return "--:--";
  const s = Math.floor(sec % 60);
  const m = Math.floor((sec / 60) % 60);
  const h = Math.floor(sec / 3600);
  if (h > 0) return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
  return `${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
}

function checksumFile(filePath, algorithm) {
  return new Promise((resolve, reject) => {
    const hash = crypto.createHash(algorithm);
    const stream = fs.createReadStream(filePath);
    stream.on("error", reject);
    stream.on("data", (chunk) => hash.update(chunk));
    stream.on("end", () => resolve(hash.digest("hex")));
  });
}

async function downloadOne(url, opts, checksumCfg, idx, totalCount) {
  const timeoutMs = Number(opts.timeout ?? 30) * 1000;
  const retries = Number(opts.retries ?? 3);
  const outputDir = path.resolve(String(opts.output ?? "."));
  fs.mkdirSync(outputDir, { recursive: true });

  const baseName = filenameFromUrl(url);
  const guessPath = path.join(outputDir, baseName);
  const existing = opts.resume && fs.existsSync(guessPath) ? fs.statSync(guessPath).size : 0;
  const headers = { ...parseHeaders(opts.header), ...(existing > 0 ? { Range: `bytes=${existing}-` } : {}) };

  let attempt = 0;
  while (attempt <= retries) {
    const startedAt = Date.now();
    let downloaded = 0;
    let total = 0;
    let finalUrl = url;
    let outPath = guessPath;
    let writer;
    let statusCode = 0;

    try {
      await new Promise((resolve, reject) => {
        const req = got.stream(url, {
          method: "GET",
          headers,
          maxRedirects: 10,
          timeout: { request: timeoutMs },
          throwHttpErrors: false,
          retry: { limit: 0 },
          https: { rejectUnauthorized: true },
        });

        req.on("response", (res) => {
          statusCode = res.statusCode ?? 0;
          if (statusCode >= 400) {
            req.destroy(new Error(`HTTP ${statusCode}`));
            return;
          }
          finalUrl = res.url || url;
          const cd = res.headers["content-disposition"];
          const name = fromContentDisposition(Array.isArray(cd) ? cd[0] : cd) || filenameFromUrl(finalUrl);
          outPath = path.join(outputDir, name);
          const isResume = opts.resume && existing > 0 && statusCode === 206 && outPath === guessPath;
          const mode = isResume ? "a" : "w";
          writer = fs.createWriteStream(outPath, { flags: mode });
          const len = Number(res.headers["content-length"] || 0);
          total = len > 0 ? len + (isResume ? existing : 0) : 0;
          downloaded = isResume ? existing : 0;
        });

        req.on("data", (chunk) => {
          downloaded += chunk.length;
          if (writer) writer.write(chunk);
          const elapsed = Math.max(0.001, (Date.now() - startedAt) / 1000);
          const speed = (downloaded - (existing > 0 ? existing : 0)) / elapsed;
          const pct = total > 0 ? (downloaded * 100) / total : 0;
          const eta = speed > 0 && total > 0 ? (total - downloaded) / speed : Number.POSITIVE_INFINITY;
          process.stdout.write(`\r[${idx + 1}/${totalCount}] ${path.basename(outPath)} | ${pct.toFixed(1)}% | ${formatBytes(downloaded)}/${total > 0 ? formatBytes(total) : "?"} | ${formatBytes(speed)}/s | ETA ${formatEta(eta)}   `);
        });

        req.on("error", (err) => {
          if (writer) writer.end();
          reject(err);
        });
        req.on("end", () => {
          if (writer) writer.end();
          resolve();
        });
      });
      process.stdout.write("\n");

      let checksumResult = "not requested";
      if (checksumCfg) {
        const gotHash = await checksumFile(outPath, checksumCfg.algorithm);
        checksumResult = gotHash.toLowerCase() === checksumCfg.expected ? "ok" : `mismatch (${gotHash})`;
      }
      const size = fs.statSync(outPath).size;
      const sec = (Date.now() - startedAt) / 1000;
      return {
        url,
        finalUrl,
        file: path.basename(outPath),
        path: outPath,
        size,
        timeSec: sec,
        avgSpeed: size / Math.max(sec, 0.001),
        checksum: checksumResult,
        statusCode,
      };
    } catch (err) {
      attempt += 1;
      if (attempt > retries) throw err;
      const waitMs = (2 ** (attempt - 1)) * 1000;
      await new Promise((r) => setTimeout(r, waitMs));
    }
  }
  throw new Error("Download failed after retries.");
}

function printSummary(rows) {
  console.log("\nDownload Summary");
  console.log("File | Size | Time | Avg Speed | Checksum | Final URL");
  console.log("---- | ---- | ---- | --------- | -------- | ---------");
  for (const r of rows) {
    console.log(`${r.file} | ${formatBytes(r.size)} | ${r.timeSec.toFixed(2)}s | ${formatBytes(r.avgSpeed)}/s | ${r.checksum} | ${r.finalUrl}`);
  }
}

async function main() {
  const { options, positional } = parseArgs(process.argv.slice(2));
  const listUrls = options.list
    ? fs.readFileSync(path.resolve(String(options.list)), "utf8").split(/\r?\n/).map((s) => s.trim()).filter(Boolean)
    : [];
  const urls = [...positional, ...listUrls];
  if (urls.length === 0) urls.push(...SAMPLE_URLS);

  const checksum = parseChecksum(options.checksum ? String(options.checksum) : "");
  const useParallel = Boolean(options.parallel);
  const threads = Math.max(1, Number(options.threads ?? 4));

  const tasks = urls.map((u, i) => async () => downloadOne(u, options, checksum && i === 0 ? checksum : null, i, urls.length));
  const results = [];

  if (useParallel && tasks.length > 1) {
    const limit = pLimit(threads);
    const settled = await Promise.all(tasks.map((t) => limit(async () => {
      try {
        const r = await t();
        results.push(r);
      } catch (e) {
        results.push({
          url: "unknown",
          finalUrl: "n/a",
          file: "n/a",
          size: 0,
          timeSec: 0,
          avgSpeed: 0,
          checksum: `failed: ${e instanceof Error ? e.message : String(e)}`,
        });
      }
    })));
    void settled;
  } else {
    for (const t of tasks) results.push(await t());
  }

  printSummary(results);
}

main().catch((e) => {
  console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
  process.exit(1);
});