File Deduplicator (cpp, written by Codex)

Codex wrote this cpp project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: File Deduplicator

Write a program that finds and manages duplicate files across directories using content-based hashing, supporting multiple deduplication strategies and detailed reporting.

FUNCTIONAL REQUIREMENTS:
- Accept one or more directory paths as command-line arguments
- Find duplicate files by comparing SHA-256 content hashes, using a two-phase approach: first compare file sizes to narrow candidates, then hash only size-matched files
- Support configurable minimum file size via --min-size flag (default: 1 byte) to skip tiny files
- Support file type filtering via --include and --exclude flags with glob patterns
- Group duplicates into sets showing all copies with their full paths, sizes, and modification dates
- Support multiple deduplication actions via --action flag: report (default, just list duplicates), delete (remove duplicates keeping the oldest/newest based on --keep flag), hardlink (replace duplicates with hard links to save space), symlink (replace with symbolic links)
- Support a --dry-run flag to preview what would be done without actually modifying files
- Scan directories recursively by default, with --no-recursive flag to disable
- Display a progress bar during scanning showing files processed and duplicates found so far
- Print summary to console: total files scanned, total unique files, duplicate sets found, total wasted space, space that would be recovered
- Save the full deduplication report as JSON with --output flag (default: dedup_report.json)
- If no directories are given, create a sample directory with intentional duplicates (exact copies, files with same content but different names, and unique files), run deduplication analysis, and display the results
- Handle errors: permission denied, broken symlinks, files modified during scan, and cross-filesystem hard links

Create a complete C++ project for a clean Ubuntu 22.04 machine with only G++ 12+ and CMake 3.22+ installed. Include:
- Source code
- CMakeLists.txt with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

cmake_minimum_required(VERSION 3.16)
project(file_deduplicator LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

add_executable(file_deduplicator src/main.cpp)

# File Deduplicator (C++)

## Requirements
- Ubuntu 22.04
- CMake 3.16+
- C++20 compiler
- `sha256sum` command available

## Build
```bash
cmake -S . -B build
cmake --build build
```

## Run
```bash
./build/file_deduplicator dir1 dir2
./build/file_deduplicator dir1 --action delete --keep oldest --dry-run
./build/file_deduplicator dir1 --action hardlink --keep newest --include "*.txt"
```

Writes JSON report to `dedup_report.json` by default.

#include <algorithm>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <map>
#include <regex>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>

namespace fs = std::filesystem;

struct Config {
    std::vector<std::string> dirs;
    std::uintmax_t minSize = 1;
    std::vector<std::string> include;
    std::vector<std::string> exclude;
    std::string action = "report";
    std::string keep = "oldest";
    bool dryRun = false;
    bool recursive = true;
    std::string output = "dedup_report.json";
};

struct FileInfo {
    fs::path path;
    std::uintmax_t size;
    std::time_t mtime;
    std::string hash;
};

static std::string quote(const std::string& s) { return "\"" + s + "\""; }

static std::string runCommand(const std::string& cmd) {
#if defined(_WIN32)
    FILE* pipe = _popen(cmd.c_str(), "r");
#else
    FILE* pipe = popen(cmd.c_str(), "r");
#endif
    if (!pipe) throw std::runtime_error("Failed to run command: " + cmd);
    std::ostringstream out;
    char buf[4096];
    while (fgets(buf, sizeof(buf), pipe)) out << buf;
#if defined(_WIN32)
    _pclose(pipe);
#else
    pclose(pipe);
#endif
    return out.str();
}

static std::string wildcardToRegex(const std::string& pat) {
    std::string r = "^";
    for (char c : pat) {
        if (c == '*') r += ".*";
        else if (c == '?') r += ".";
        else if (std::string(".^$|()[]{}+\\").find(c) != std::string::npos) { r += '\\'; r += c; }
        else r += c;
    }
    r += "$";
    return r;
}

static bool matches(const std::string& name, const std::vector<std::string>& pats, bool defaultIfEmpty) {
    if (pats.empty()) return defaultIfEmpty;
    for (const auto& p : pats) {
        std::regex rx(wildcardToRegex(p));
        if (std::regex_match(name, rx)) return true;
    }
    return false;
}

static std::string sha256(const fs::path& p) {
    std::string out = runCommand("sha256sum " + quote(p.string()));
    std::stringstream ss(out);
    std::string hash;
    ss >> hash;
    return hash;
}

static Config parseArgs(int argc, char** argv) {
    Config cfg;
    for (int i = 1; i < argc; ++i) {
        std::string a = argv[i];
        if (!a.starts_with("--")) {
            cfg.dirs.push_back(a);
            continue;
        }
        if (a == "--min-size" && i + 1 < argc) cfg.minSize = std::stoull(argv[++i]);
        else if (a == "--include" && i + 1 < argc) cfg.include.push_back(argv[++i]);
        else if (a == "--exclude" && i + 1 < argc) cfg.exclude.push_back(argv[++i]);
        else if (a == "--action" && i + 1 < argc) cfg.action = argv[++i];
        else if (a == "--keep" && i + 1 < argc) cfg.keep = argv[++i];
        else if (a == "--dry-run") cfg.dryRun = true;
        else if (a == "--no-recursive") cfg.recursive = false;
        else if (a == "--output" && i + 1 < argc) cfg.output = argv[++i];
        else throw std::runtime_error("Unknown option: " + a);
    }
    return cfg;
}

static std::vector<std::string> makeSample() {
    fs::path root = "sample_dedup_data";
    fs::remove_all(root);
    fs::create_directories(root / "a");
    fs::create_directories(root / "b");
    {
        std::ofstream(root / "a" / "x1.txt") << "hello duplicate\n";
        std::ofstream(root / "a" / "x2.txt") << "hello duplicate\n";
        std::ofstream(root / "b" / "x3.txt") << "hello duplicate\n";
        std::ofstream(root / "b" / "unique.txt") << "unique\n";
    }
    return {root.string()};
}

static std::vector<FileInfo> scan(const Config& cfg) {
    std::vector<FileInfo> files;
    std::size_t seen = 0;
    for (const auto& d : cfg.dirs) {
        fs::path root = d;
        if (!fs::exists(root)) {
            std::cerr << "Warning: missing directory " << d << "\n";
            continue;
        }
        if (cfg.recursive) {
            for (auto it = fs::recursive_directory_iterator(root); it != fs::recursive_directory_iterator(); ++it) {
                if (!it->is_regular_file()) continue;
                auto p = it->path();
                auto size = fs::file_size(p);
                if (size < cfg.minSize) continue;
                std::string name = p.filename().string();
                if (!matches(name, cfg.include, true) || matches(name, cfg.exclude, false)) continue;
                auto ftime = fs::last_write_time(p);
                auto cftime = decltype(ftime)::clock::to_time_t(ftime);
                files.push_back({fs::absolute(p), size, cftime, ""});
                seen++;
                if (seen % 250 == 0) std::cout << "\rScanned " << seen << " files..." << std::flush;
            }
        } else {
            for (auto it = fs::directory_iterator(root); it != fs::directory_iterator(); ++it) {
                if (!it->is_regular_file()) continue;
                auto p = it->path();
                auto size = fs::file_size(p);
                if (size < cfg.minSize) continue;
                std::string name = p.filename().string();
                if (!matches(name, cfg.include, true) || matches(name, cfg.exclude, false)) continue;
                auto ftime = fs::last_write_time(p);
                auto cftime = decltype(ftime)::clock::to_time_t(ftime);
                files.push_back({fs::absolute(p), size, cftime, ""});
                seen++;
            }
        }
    }
    std::cout << "\n";
    return files;
}

static std::string escapeJson(std::string s) {
    std::string out;
    for (char c : s) {
        if (c == '\\') out += "\\\\";
        else if (c == '"') out += "\\\"";
        else if (c == '\n') out += "\\n";
        else out += c;
    }
    return out;
}

int main(int argc, char** argv) {
    try {
        Config cfg = parseArgs(argc, argv);
        if (cfg.dirs.empty()) cfg.dirs = makeSample();

        auto files = scan(cfg);

        std::map<std::uintmax_t, std::vector<FileInfo>> bySize;
        for (const auto& f : files) bySize[f.size].push_back(f);

        struct DupSet { std::uintmax_t size; std::string hash; std::vector<FileInfo> files; };
        std::vector<DupSet> sets;

        for (auto& kv : bySize) {
            if (kv.second.size() < 2) continue;
            std::map<std::string, std::vector<FileInfo>> byHash;
            for (auto& fi : kv.second) {
                fi.hash = sha256(fi.path);
                byHash[fi.hash].push_back(fi);
            }
            for (auto& h : byHash) {
                if (h.second.size() > 1) sets.push_back({kv.first, h.first, h.second});
            }
        }

        std::uintmax_t wasted = 0;
        std::uintmax_t recovered = 0;
        std::vector<std::string> actionRows;

        for (auto& ds : sets) {
            auto sorted = ds.files;
            std::sort(sorted.begin(), sorted.end(), [](const FileInfo& a, const FileInfo& b){ return a.mtime < b.mtime; });
            FileInfo keeper = cfg.keep == "oldest" ? sorted.front() : sorted.back();
            wasted += ds.size * (sorted.size() - 1);

            for (const auto& fi : sorted) {
                if (fi.path == keeper.path) continue;
                std::string status = "planned";
                std::string err;
                if (cfg.action != "report") {
                    if (cfg.dryRun) {
                        status = "dry-run";
                        recovered += fi.size;
                    } else {
                        try {
                            if (cfg.action == "delete") {
                                fs::remove(fi.path);
                            } else if (cfg.action == "hardlink") {
                                fs::remove(fi.path);
                                fs::create_hard_link(keeper.path, fi.path);
                            } else if (cfg.action == "symlink") {
                                fs::remove(fi.path);
                                fs::create_symlink(fs::relative(keeper.path, fi.path.parent_path()), fi.path);
                            }
                            status = "done";
                            recovered += fi.size;
                        } catch (const std::exception& ex) {
                            status = "failed";
                            err = ex.what();
                        }
                    }
                }
                std::ostringstream row;
                row << "{\"type\":\"" << escapeJson(cfg.action) << "\",\"source\":\"" << escapeJson(keeper.path.string())
                    << "\",\"target\":\"" << escapeJson(fi.path.string()) << "\",\"status\":\"" << status << "\"";
                if (!err.empty()) row << ",\"error\":\"" << escapeJson(err) << "\"";
                row << "}";
                actionRows.push_back(row.str());
            }
        }

        std::uintmax_t dupFiles = 0;
        for (const auto& ds : sets) dupFiles += ds.files.size();
        std::int64_t uniqueEstimate = static_cast<std::int64_t>(files.size()) - static_cast<std::int64_t>(dupFiles - sets.size());

        std::ostringstream json;
        json << "{\n"
             << "  \"summary\": {\n"
             << "    \"totalFilesScanned\": " << files.size() << ",\n"
             << "    \"duplicateSetsFound\": " << sets.size() << ",\n"
             << "    \"totalWastedSpace\": " << wasted << ",\n"
             << "    \"recoverableSpace\": " << (recovered ? recovered : wasted) << ",\n"
             << "    \"totalUniqueFilesEstimate\": " << uniqueEstimate << "\n"
             << "  },\n"
             << "  \"duplicateSets\": [\n";

        for (std::size_t i = 0; i < sets.size(); ++i) {
            const auto& ds = sets[i];
            json << "    {\"size\":" << ds.size << ",\"hash\":\"" << escapeJson(ds.hash) << "\",\"files\":[";
            for (std::size_t j = 0; j < ds.files.size(); ++j) {
                const auto& fi = ds.files[j];
                json << "{\"path\":\"" << escapeJson(fi.path.string()) << "\",\"size\":" << fi.size << ",\"mtime\":" << fi.mtime << ",\"hash\":\"" << escapeJson(fi.hash) << "\"}";
                if (j + 1 < ds.files.size()) json << ",";
            }
            json << "]}";
            if (i + 1 < sets.size()) json << ",";
            json << "\n";
        }
        json << "  ],\n  \"actions\": [\n";
        for (std::size_t i = 0; i < actionRows.size(); ++i) {
            json << "    " << actionRows[i];
            if (i + 1 < actionRows.size()) json << ",";
            json << "\n";
        }
        json << "  ]\n}\n";

        std::ofstream out(cfg.output);
        out << json.str();

        std::cout << "Duplicate sets: " << sets.size() << "\n";
        std::cout << "Wasted space: " << wasted << " bytes\n";
        std::cout << "Recoverable: " << (recovered ? recovered : wasted) << " bytes\n";
    } catch (const std::exception& ex) {
        std::cerr << "Error: " << ex.what() << "\n";
        return 1;
    }
    return 0;
}