File Deduplicator (javascript, written by Codex)

Codex wrote this javascript project from the task below. It installed and ran on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: File Deduplicator

Write a program that finds and manages duplicate files across directories using content-based hashing, supporting multiple deduplication strategies and detailed reporting.

FUNCTIONAL REQUIREMENTS:
- Accept one or more directory paths as command-line arguments
- Find duplicate files by comparing SHA-256 content hashes, using a two-phase approach: first compare file sizes to narrow candidates, then hash only size-matched files
- Support configurable minimum file size via --min-size flag (default: 1 byte) to skip tiny files
- Support file type filtering via --include and --exclude flags with glob patterns
- Group duplicates into sets showing all copies with their full paths, sizes, and modification dates
- Support multiple deduplication actions via --action flag: report (default, just list duplicates), delete (remove duplicates keeping the oldest/newest based on --keep flag), hardlink (replace duplicates with hard links to save space), symlink (replace with symbolic links)
- Support a --dry-run flag to preview what would be done without actually modifying files
- Scan directories recursively by default, with --no-recursive flag to disable
- Display a progress bar during scanning showing files processed and duplicates found so far
- Print summary to console: total files scanned, total unique files, duplicate sets found, total wasted space, space that would be recovered
- Save the full deduplication report as JSON with --output flag (default: dedup_report.json)
- If no directories are given, create a sample directory with intentional duplicates (exact copies, files with same content but different names, and unique files), run deduplication analysis, and display the results
- Handle errors: permission denied, broken symlinks, files modified during scan, and cross-filesystem hard links

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

{
  "name": "file-deduplicator",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "file-deduplicator",
      "version": "1.0.0",
      "license": "MIT",
      "dependencies": {
        "minimatch": "10.0.1"
      },
      "engines": {
        "node": ">=20.0.0"
      }
    },
    "node_modules/balanced-match": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
      "license": "MIT"
    },
    "node_modules/brace-expansion": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
      "license": "MIT",
      "dependencies": {
        "balanced-match": "^1.0.0"
      }
    },
    "node_modules/minimatch": {
      "version": "10.0.1",
      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz",
      "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==",
      "license": "ISC",
      "dependencies": {
        "brace-expansion": "^2.0.1"
      },
      "engines": {
        "node": "20 || >=22"
      },
      "funding": {
        "url": "https://github.com/sponsors/isaacs"
      }
    }
  }
}

{
  "name": "file-deduplicator",
  "version": "1.0.0",
  "description": "Find and manage duplicate files",
  "type": "module",
  "main": "src/index.js",
  "scripts": { "start": "node src/index.js" },
  "engines": { "node": ">=20.0.0" },
  "dependencies": {
    "minimatch": "10.0.1"
  },
  "license": "MIT"
}

# File Deduplicator (JavaScript)

## Requirements
- Ubuntu 22.04
- Node.js 20+

## Install
```bash
npm install
```

## Run
```bash
node src/index.js dir1 dir2
node src/index.js dir1 --action delete --keep oldest --dry-run
node src/index.js dir1 --action hardlink --keep newest --include "*.txt" --exclude "*.log"
```

## Output
- Console summary with duplicates and recoverable bytes
- JSON report file (default `dedup_report.json`, configurable with `--output`)

If no dirs are provided, sample duplicate data is generated and analyzed.

import fs from "node:fs";
import path from "node:path";
import crypto from "node:crypto";
import process from "node:process";
import { minimatch } from "minimatch";

function parseArgs(argv) {
  const cfg = {
    dirs: [],
    minSize: 1,
    include: [],
    exclude: [],
    action: "report",
    keep: "oldest",
    dryRun: false,
    recursive: true,
    output: "dedup_report.json"
  };
  for (let i = 0; i < argv.length; i += 1) {
    const a = argv[i];
    if (!a.startsWith("--")) {
      cfg.dirs.push(a);
      continue;
    }
    if (a === "--min-size") cfg.minSize = Number.parseInt(argv[++i], 10);
    else if (a === "--include") cfg.include.push(argv[++i]);
    else if (a === "--exclude") cfg.exclude.push(argv[++i]);
    else if (a === "--action") cfg.action = argv[++i];
    else if (a === "--keep") cfg.keep = argv[++i];
    else if (a === "--dry-run") cfg.dryRun = true;
    else if (a === "--no-recursive") cfg.recursive = false;
    else if (a === "--output") cfg.output = argv[++i];
    else throw new Error(`Unknown option: ${a}`);
  }
  if (!["report", "delete", "hardlink", "symlink"].includes(cfg.action)) throw new Error("--action must be report|delete|hardlink|symlink");
  if (!["oldest", "newest"].includes(cfg.keep)) throw new Error("--keep must be oldest|newest");
  return cfg;
}

function sha256(filePath) {
  const h = crypto.createHash("sha256");
  h.update(fs.readFileSync(filePath));
  return h.digest("hex");
}

function shouldInclude(rel, cfg) {
  const inc = cfg.include.length === 0 || cfg.include.some((p) => minimatch(rel, p, { dot: true }));
  const exc = cfg.exclude.some((p) => minimatch(rel, p, { dot: true }));
  return inc && !exc;
}

function walk(dir, recursive) {
  const out = [];
  const stack = [path.resolve(dir)];
  while (stack.length) {
    const cur = stack.pop();
    const ents = fs.readdirSync(cur, { withFileTypes: true });
    for (const e of ents) {
      const full = path.join(cur, e.name);
      if (e.isDirectory()) {
        if (recursive) stack.push(full);
      } else if (e.isFile()) {
        out.push(full);
      }
    }
  }
  return out;
}

function scan(cfg) {
  const files = [];
  for (const d of cfg.dirs) {
    if (!fs.existsSync(d)) {
      process.stderr.write(`Warning: missing directory ${d}\n`);
      continue;
    }
    files.push(...walk(d, cfg.recursive));
  }

  const bySize = new Map();
  let processed = 0;
  for (const f of files) {
    try {
      const st = fs.statSync(f);
      if (st.size < cfg.minSize) continue;
      const rel = path.basename(f);
      if (!shouldInclude(rel, cfg)) continue;
      if (!bySize.has(st.size)) bySize.set(st.size, []);
      bySize.get(st.size).push({ path: f, size: st.size, mtimeMs: st.mtimeMs });
      processed += 1;
      if (processed % 250 === 0 || processed === files.length) process.stdout.write(`\rScanned ${processed}/${files.length} files...`);
    } catch (err) {
      process.stderr.write(`Warning: ${f}: ${err.message}\n`);
    }
  }
  process.stdout.write("\n");

  const dupSets = [];
  for (const [size, group] of bySize.entries()) {
    if (group.length < 2) continue;
    const byHash = new Map();
    for (const item of group) {
      try {
        const h = sha256(item.path);
        if (!byHash.has(h)) byHash.set(h, []);
        byHash.get(h).push({ ...item, hash: h });
      } catch (err) {
        process.stderr.write(`Warning: hash failed ${item.path}: ${err.message}\n`);
      }
    }
    for (const arr of byHash.values()) {
      if (arr.length > 1) dupSets.push({ size, hash: arr[0].hash, files: arr });
    }
  }
  return { filesScanned: processed, duplicateSets: dupSets };
}

function chooseKeep(files, keep) {
  const sorted = [...files].sort((a, b) => a.mtimeMs - b.mtimeMs);
  return keep === "oldest" ? sorted[0] : sorted[sorted.length - 1];
}

function applyActions(dupSets, cfg) {
  let reclaimed = 0;
  const actions = [];

  for (const set of dupSets) {
    const keeper = chooseKeep(set.files, cfg.keep);
    const targets = set.files.filter((f) => f.path !== keeper.path);

    for (const t of targets) {
      const action = { type: cfg.action, source: keeper.path, target: t.path, status: "planned" };
      if (cfg.action === "report") {
        actions.push(action);
        continue;
      }

      if (cfg.dryRun) {
        actions.push({ ...action, status: "dry-run" });
        reclaimed += t.size;
        continue;
      }

      try {
        if (cfg.action === "delete") {
          fs.unlinkSync(t.path);
        } else if (cfg.action === "hardlink") {
          fs.unlinkSync(t.path);
          fs.linkSync(keeper.path, t.path);
        } else if (cfg.action === "symlink") {
          fs.unlinkSync(t.path);
          fs.symlinkSync(path.relative(path.dirname(t.path), keeper.path), t.path);
        }
        actions.push({ ...action, status: "done" });
        reclaimed += t.size;
      } catch (err) {
        actions.push({ ...action, status: "failed", error: err.message });
      }
    }
  }

  return { actions, reclaimedBytes: reclaimed };
}

function makeSample() {
  const root = path.resolve("sample_dedup_data");
  fs.rmSync(root, { recursive: true, force: true });
  fs.mkdirSync(path.join(root, "a"), { recursive: true });
  fs.mkdirSync(path.join(root, "b"), { recursive: true });

  fs.writeFileSync(path.join(root, "a", "x1.txt"), "hello duplicate\n", "utf8");
  fs.writeFileSync(path.join(root, "a", "x2.txt"), "hello duplicate\n", "utf8");
  fs.writeFileSync(path.join(root, "b", "x3.txt"), "hello duplicate\n", "utf8");
  fs.writeFileSync(path.join(root, "b", "unique.txt"), "unique data\n", "utf8");

  return [root];
}

function main() {
  try {
    const cfg = parseArgs(process.argv.slice(2));
    if (!cfg.dirs.length) cfg.dirs = makeSample();

    const scanRes = scan(cfg);
    const actionRes = applyActions(scanRes.duplicateSets, cfg);

    const totalDupFiles = scanRes.duplicateSets.reduce((s, ds) => s + ds.files.length, 0);
    const wasted = scanRes.duplicateSets.reduce((s, ds) => s + ds.size * (ds.files.length - 1), 0);

    const report = {
      generatedAt: new Date().toISOString(),
      config: cfg,
      summary: {
        totalFilesScanned: scanRes.filesScanned,
        totalUniqueFilesEstimate: scanRes.filesScanned - (totalDupFiles - scanRes.duplicateSets.length),
        duplicateSetsFound: scanRes.duplicateSets.length,
        totalWastedSpace: wasted,
        recoverableSpace: actionRes.reclaimedBytes || wasted
      },
      duplicateSets: scanRes.duplicateSets,
      actions: actionRes.actions
    };

    process.stdout.write(`Duplicate sets: ${report.summary.duplicateSetsFound}\n`);
    process.stdout.write(`Wasted space: ${report.summary.totalWastedSpace} bytes\n`);
    process.stdout.write(`Recoverable: ${report.summary.recoverableSpace} bytes\n`);

    fs.writeFileSync(path.resolve(cfg.output), JSON.stringify(report, null, 2), "utf8");
  } catch (err) {
    process.stderr.write(`Error: ${err.message}\n`);
    process.exit(1);
  }
}

main();