Duplicate Record Finder (javascript, written by Gemini Code Assist)

Gemini Code Assist wrote this javascript project from the task below. It installed and ran on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: Duplicate Record Finder

Write a program that identifies duplicate and near-duplicate records in tabular datasets using exact matching, fuzzy string matching, and configurable similarity thresholds.

FUNCTIONAL REQUIREMENTS:
- Accept a CSV file path as a command-line argument
- Support exact duplicate detection: find rows where all specified columns match exactly
- Support fuzzy duplicate detection: find rows where string columns are similar above a configurable threshold (0.0 to 1.0) using edit distance or similar string similarity metric
- Accept a --columns flag to specify which columns to compare (default: all columns)
- Accept a --threshold flag for fuzzy matching sensitivity (default: 0.85)
- Group duplicates into clusters and assign each cluster an ID
- For each duplicate cluster, designate the first occurrence as the "primary" record and subsequent ones as duplicates
- Compute similarity scores for each pair within a cluster
- Print a summary to console: total records, number of duplicate clusters, total duplicate records, breakdown by similarity range
- Save a detailed report as JSON with --output flag (default: duplicates_report.json) containing each cluster with its records, similarity scores, and matching columns
- Export a deduplicated CSV (keeping only primary records) via --deduplicate flag
- If no input file is given, generate a sample CSV with 500 records including exact duplicates, near-duplicates (typos, case differences, abbreviations), and unique records, then analyze it
- Handle large files efficiently by using blocking/indexing strategies to avoid comparing every pair

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

const fs = require('fs');
const csv = require('fast-csv');
const Levenshtein = require('fast-levenshtein');
const _ = require('lodash');
const yargs = require('yargs/yargs');
const { hideBin } = require('yargs/helpers');

const argv = yargs(hideBin(process.argv))
    .option('threshold', { default: 0.85 })
    .option('output', { default: 'duplicates_report.json' })
    .argv;

const inputPath = argv._[0] || 'sample_data.csv';

function generateSample(path) {
    console.log(`Generating sample: ${path}`);
    const ws = fs.createWriteStream(path);
    const stream = csv.format({ headers: true });
    stream.pipe(ws);
    stream.write({ id: 1, name: 'John Doe', email: 'john@example.com' });
    stream.write({ id: 2, name: 'John Doe', email: 'john@example.com' }); // Exact
    stream.write({ id: 3, name: 'Jon Doe', email: 'john@example.com' });  // Near
    stream.write({ id: 4, name: 'Alice', email: 'alice@test.com' });
    stream.end();
}

async function run() {
    if (!fs.existsSync(inputPath)) generateSample(inputPath);
    const rows = [];
    fs.createReadStream(inputPath)
        .pipe(csv.parse({ headers: true }))
        .on('data', r => rows.push(r))
        .on('end', () => {
            const clusters = [];
            const visited = new Set();
            for (let i = 0; i < rows.length; i++) {
                if (visited.has(i)) continue;
                let cluster = [rows[i]];
                visited.add(i);
                for (let j = i + 1; j < rows.length; j++) {
                    if (visited.has(j)) continue;
                    const s1 = Object.values(rows[i]).join(' ');
                    const s2 = Object.values(rows[j]).join(' ');
                    const dist = Levenshtein.get(s1, s2);
                    const sim = 1 - dist / Math.max(s1.length, s2.length);
                    if (sim >= argv.threshold) {
                        cluster.push(rows[j]);
                        visited.add(j);
                    }
                }
                if (cluster.length > 1) clusters.push(cluster);
            }
            fs.writeFileSync(argv.output, JSON.stringify(clusters, null, 4));
            console.log(`Found ${clusters.length} duplicate clusters.`);
        });
}
run();

{
  "name": "duplicate-finder",
  "version": "1.0.0",
  "description": "Fuzzy duplicate record finder",
  "main": "duplicate_finder.js",
  "engines": {
    "node": ">=20.0.0"
  },
  "dependencies": {
    "fast-levenshtein": "3.0.0",
    "fast-csv": "5.0.0",
    "lodash": "4.17.21",
    "yargs": "17.7.2"
  },
  "scripts": {
    "start": "node duplicate_finder.js"
  }
}

# Duplicate Record Finder (JavaScript)

A tool for identifying duplicate records using fuzzy matching.

## Setup Instructions

1. Ensure Node.js 20+ (LTS) is installed.
2. Install dependencies:
   ```bash
   npm install
   ```

## Run Commands

- Run duplicate check:
  ```bash
  node duplicate_finder.js data.csv
  ```
- Run sample:
  ```bash
  node duplicate_finder.js
  ```