Duplicate Record Finder (javascript, written by Claude Code)

Claude Code wrote this javascript project from the task below. It installed and ran on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: Duplicate Record Finder

Write a program that identifies duplicate and near-duplicate records in tabular datasets using exact matching, fuzzy string matching, and configurable similarity thresholds.

FUNCTIONAL REQUIREMENTS:
- Accept a CSV file path as a command-line argument
- Support exact duplicate detection: find rows where all specified columns match exactly
- Support fuzzy duplicate detection: find rows where string columns are similar above a configurable threshold (0.0 to 1.0) using edit distance or similar string similarity metric
- Accept a --columns flag to specify which columns to compare (default: all columns)
- Accept a --threshold flag for fuzzy matching sensitivity (default: 0.85)
- Group duplicates into clusters and assign each cluster an ID
- For each duplicate cluster, designate the first occurrence as the "primary" record and subsequent ones as duplicates
- Compute similarity scores for each pair within a cluster
- Print a summary to console: total records, number of duplicate clusters, total duplicate records, breakdown by similarity range
- Save a detailed report as JSON with --output flag (default: duplicates_report.json) containing each cluster with its records, similarity scores, and matching columns
- Export a deduplicated CSV (keeping only primary records) via --deduplicate flag
- If no input file is given, generate a sample CSV with 500 records including exact duplicates, near-duplicates (typos, case differences, abbreviations), and unique records, then analyze it
- Handle large files efficiently by using blocking/indexing strategies to avoid comparing every pair

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

#!/usr/bin/env node
/**
 * Duplicate Record Finder - Trial 1
 * Uses fuse.js + csv-parse + cli-table3.
 * Supports exact matching, fuzzy string matching with configurable thresholds,
 * and multiple comparison strategies (Levenshtein, Jaro-Winkler, Cosine).
 */

const fs = require('fs');
const path = require('path');
const { parse } = require('csv-parse/sync');
const { stringify } = require('csv-stringify/sync');
const Fuse = require('fuse.js');
const Table = require('cli-table3');

// --- Similarity Strategies ---

function levenshteinDistance(s1, s2) {
    const m = s1.length, n = s2.length;
    const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
    for (let i = 0; i <= m; i++) dp[i][0] = i;
    for (let j = 0; j <= n; j++) dp[0][j] = j;
    for (let i = 1; i <= m; i++) {
        for (let j = 1; j <= n; j++) {
            dp[i][j] = s1[i - 1] === s2[j - 1]
                ? dp[i - 1][j - 1]
                : 1 + Math.min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]);
        }
    }
    return dp[m][n];
}

function levenshteinSimilarity(s1, s2) {
    s1 = String(s1 || '');
    s2 = String(s2 || '');
    if (!s1 && !s2) return 1.0;
    const maxLen = Math.max(s1.length, s2.length);
    if (maxLen === 0) return 1.0;
    return 1.0 - levenshteinDistance(s1, s2) / maxLen;
}

function jaroSimilarity(s1, s2) {
    s1 = String(s1 || '');
    s2 = String(s2 || '');
    if (s1 === s2) return 1.0;
    if (!s1.length || !s2.length) return 0.0;

    const matchDist = Math.max(Math.floor(Math.max(s1.length, s2.length) / 2) - 1, 0);
    const s1Matches = Array(s1.length).fill(false);
    const s2Matches = Array(s2.length).fill(false);
    let matches = 0, transpositions = 0;

    for (let i = 0; i < s1.length; i++) {
        const start = Math.max(0, i - matchDist);
        const end = Math.min(i + matchDist + 1, s2.length);
        for (let j = start; j < end; j++) {
            if (s2Matches[j] || s1[i] !== s2[j]) continue;
            s1Matches[i] = true;
            s2Matches[j] = true;
            matches++;
            break;
        }
    }

    if (matches === 0) return 0.0;

    let k = 0;
    for (let i = 0; i < s1.length; i++) {
        if (!s1Matches[i]) continue;
        while (!s2Matches[k]) k++;
        if (s1[i] !== s2[k]) transpositions++;
        k++;
    }

    return (matches / s1.length + matches / s2.length + (matches - transpositions / 2) / matches) / 3;
}

function jaroWinklerSimilarity(s1, s2) {
    const jaro = jaroSimilarity(s1, s2);
    s1 = String(s1 || '');
    s2 = String(s2 || '');
    let prefix = 0;
    for (let i = 0; i < Math.min(4, Math.min(s1.length, s2.length)); i++) {
        if (s1[i] === s2[i]) prefix++;
        else break;
    }
    return jaro + prefix * 0.1 * (1 - jaro);
}

function cosineSimilarity(s1, s2) {
    s1 = String(s1 || '').toLowerCase();
    s2 = String(s2 || '').toLowerCase();
    if (!s1 || !s2) return 0.0;

    const bigrams = (s) => {
        const bg = {};
        for (let i = 0; i < s.length - 1; i++) {
            const key = s.substring(i, i + 2);
            bg[key] = (bg[key] || 0) + 1;
        }
        return bg;
    };

    const bg1 = bigrams(s1);
    const bg2 = bigrams(s2);
    if (!Object.keys(bg1).length || !Object.keys(bg2).length) return s1 === s2 ? 1.0 : 0.0;

    const allKeys = new Set([...Object.keys(bg1), ...Object.keys(bg2)]);
    let dot = 0, mag1 = 0, mag2 = 0;
    for (const k of allKeys) {
        const v1 = bg1[k] || 0, v2 = bg2[k] || 0;
        dot += v1 * v2;
        mag1 += v1 * v1;
        mag2 += v2 * v2;
    }
    mag1 = Math.sqrt(mag1);
    mag2 = Math.sqrt(mag2);
    if (mag1 === 0 || mag2 === 0) return 0.0;
    return dot / (mag1 * mag2);
}

const STRATEGIES = {
    'levenshtein': levenshteinSimilarity,
    'jaro-winkler': jaroWinklerSimilarity,
    'cosine': cosineSimilarity,
};

// --- Core logic ---

function computeRecordSimilarity(rec1, rec2, columnRules, defaultStrategy) {
    const columnScores = {};
    let weightedScore = 0, weightsSum = 0;

    for (const [col, rule] of Object.entries(columnRules)) {
        const strategyName = rule.strategy || defaultStrategy;
        const weight = rule.weight || 1.0;
        const strategyFn = STRATEGIES[strategyName] || levenshteinSimilarity;

        const val1 = String(rec1[col] || '').trim();
        const val2 = String(rec2[col] || '').trim();

        let score;
        if (rule.exact) {
            score = val1.toLowerCase() === val2.toLowerCase() ? 1.0 : 0.0;
        } else {
            score = strategyFn(val1, val2);
        }

        columnScores[col] = { score: Math.round(score * 10000) / 10000, strategy: strategyName };
        weightedScore += score * weight;
        weightsSum += weight;
    }

    const overall = weightsSum > 0 ? Math.round((weightedScore / weightsSum) * 10000) / 10000 : 0;
    return { overall, columns: columnScores };
}

function findExactDuplicates(records) {
    const groups = {};
    records.forEach((rec, i) => {
        const key = Object.entries(rec).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join('|');
        if (!groups[key]) groups[key] = [];
        groups[key].push(i);
    });
    return Object.values(groups).filter(g => g.length > 1);
}

function findFuzzyDuplicates(records, columnRules, threshold, defaultStrategy) {
    const n = records.length;
    const parent = Array.from({ length: n }, (_, i) => i);

    function find(x) {
        while (parent[x] !== x) { parent[x] = parent[parent[x]]; x = parent[x]; }
        return x;
    }
    function union(a, b) {
        const ra = find(a), rb = find(b);
        if (ra !== rb) parent[ra] = rb;
    }

    const pairScores = {};

    for (let i = 0; i < n; i++) {
        for (let j = i + 1; j < n; j++) {
            const result = computeRecordSimilarity(records[i], records[j], columnRules, defaultStrategy);
            if (result.overall >= threshold) {
                union(i, j);
                pairScores[`${i}-${j}`] = result;
            }
        }
    }

    const groupMap = {};
    for (let i = 0; i < n; i++) {
        const root = find(i);
        if (!groupMap[root]) groupMap[root] = [];
        groupMap[root].push(i);
    }

    const duplicateGroups = [];
    for (const members of Object.values(groupMap)) {
        if (members.length > 1) {
            const gpScores = {};
            for (let a = 0; a < members.length; a++) {
                for (let b = a + 1; b < members.length; b++) {
                    const key = `${Math.min(members[a], members[b])}-${Math.max(members[a], members[b])}`;
                    if (pairScores[key]) gpScores[key] = pairScores[key];
                }
            }
            duplicateGroups.push({ indices: members.sort((a, b) => a - b), pairScores: gpScores });
        }
    }
    return duplicateGroups;
}

function buildColumnRules(columns, strategy, exactCols) {
    const rules = {};
    for (const col of columns) {
        if (col.toLowerCase() === 'id' || col.toLowerCase() === 'index') continue;
        rules[col] = { strategy, weight: 1.0, exact: exactCols.includes(col) };
    }
    return rules;
}

// --- Sample data ---

function generateSampleDataset(outputPath) {
    const records = [
        { id: '1', first_name: 'John', last_name: 'Smith', email: 'john.smith@email.com', phone: '555-0101', city: 'New York' },
        { id: '2', first_name: 'John', last_name: 'Smith', email: 'john.smith@email.com', phone: '555-0101', city: 'New York' },
        { id: '3', first_name: 'Jon', last_name: 'Smyth', email: 'jon.smyth@email.com', phone: '555-0101', city: 'New York' },
        { id: '4', first_name: 'Jane', last_name: 'Doe', email: 'jane.doe@email.com', phone: '555-0202', city: 'Los Angeles' },
        { id: '5', first_name: 'Jane', last_name: 'Doe', email: 'jane.doe@email.com', phone: '555-0202', city: 'Los Angeles' },
        { id: '6', first_name: 'Jayne', last_name: 'Doe', email: 'jayne.doe@email.com', phone: '555-0203', city: 'Los Angeles' },
        { id: '7', first_name: 'Robert', last_name: 'Johnson', email: 'r.johnson@email.com', phone: '555-0303', city: 'Chicago' },
        { id: '8', first_name: 'Bob', last_name: 'Johnson', email: 'bob.johnson@email.com', phone: '555-0304', city: 'Chicago' },
        { id: '9', first_name: 'Alice', last_name: 'Williams', email: 'alice.w@email.com', phone: '555-0404', city: 'Houston' },
        { id: '10', first_name: 'Alice', last_name: 'Willams', email: 'alice.w@email.com', phone: '555-0404', city: 'Houston' },
        { id: '11', first_name: 'Michael', last_name: 'Brown', email: 'm.brown@email.com', phone: '555-0505', city: 'Phoenix' },
        { id: '12', first_name: 'Emily', last_name: 'Davis', email: 'emily.d@email.com', phone: '555-0606', city: 'Philadelphia' },
        { id: '13', first_name: 'Emilie', last_name: 'Davis', email: 'emilie.davis@email.com', phone: '555-0607', city: 'Philadelphia' },
        { id: '14', first_name: 'David', last_name: 'Garcia', email: 'd.garcia@email.com', phone: '555-0707', city: 'San Antonio' },
        { id: '15', first_name: 'David', last_name: 'Garcia', email: 'd.garcia@email.com', phone: '555-0707', city: 'San Antonio' },
    ];

    const csv = stringify(records, { header: true });
    fs.writeFileSync(outputPath, csv);
    console.log(`Sample dataset generated: ${outputPath} (${records.length} records)`);
    return outputPath;
}

// --- Reporting ---

function printConsoleReport(records, exactGroups, fuzzyGroups) {
    console.log('\n' + '='.repeat(70));
    console.log('  DUPLICATE RECORD FINDER - REPORT (fuse.js + csv-parse + cli-table3)');
    console.log('='.repeat(70));
    console.log(`\nTotal records analyzed: ${records.length}`);

    console.log(`\n--- Exact Duplicates: ${exactGroups.length} group(s) ---`);
    exactGroups.forEach((group, i) => {
        console.log(`\n  Group ${i + 1} (${group.length} records):`);

        const table = new Table({
            head: Object.keys(records[0]),
            style: { head: ['cyan'] },
        });
        group.forEach(idx => {
            table.push([`Row ${idx}`, ...Object.values(records[idx]).slice(0)]);
        });
        // Rebuild table with row index
        const table2 = new Table({
            head: ['Row', ...Object.keys(records[0])],
        });
        group.forEach(idx => {
            table2.push([idx, ...Object.values(records[idx])]);
        });
        console.log(table2.toString());
    });

    console.log(`\n--- Fuzzy Duplicate Groups: ${fuzzyGroups.length} group(s) ---`);
    fuzzyGroups.forEach((group, i) => {
        console.log(`\n  Group ${i + 1} (${group.indices.length} records):`);
        const table = new Table({
            head: ['Row', ...Object.keys(records[0])],
        });
        group.indices.forEach(idx => {
            table.push([idx, ...Object.values(records[idx])]);
        });
        console.log(table.toString());

        for (const [pairKey, scores] of Object.entries(group.pairScores)) {
            console.log(`    Pair ${pairKey}: overall=${scores.overall}`);
            for (const [col, info] of Object.entries(scores.columns)) {
                console.log(`      ${col}: ${info.score} (${info.strategy})`);
            }
        }
    });

    console.log('\n' + '='.repeat(70));
}

function generateJsonReport(records, exactGroups, fuzzyGroups, outputPath) {
    const report = {
        summary: {
            total_records: records.length,
            exact_duplicate_groups: exactGroups.length,
            fuzzy_duplicate_groups: fuzzyGroups.length,
        },
        exact_duplicates: exactGroups.map(group => ({
            records: group.map(idx => ({ row_index: idx, data: records[idx] })),
        })),
        fuzzy_duplicates: fuzzyGroups.map(group => ({
            records: group.indices.map(idx => ({ row_index: idx, data: records[idx] })),
            pair_scores: group.pairScores,
        })),
    };

    fs.writeFileSync(outputPath, JSON.stringify(report, null, 2));
    console.log(`\nJSON report saved to: ${outputPath}`);
    return report;
}

// --- Main ---

function parseArgs() {
    const args = process.argv.slice(2);
    const config = { input: null, threshold: 0.8, strategy: 'levenshtein', exactCols: [], output: 'duplicates_report.json' };

    for (let i = 0; i < args.length; i++) {
        switch (args[i]) {
            case '--input': case '-i': config.input = args[++i]; break;
            case '--threshold': case '-t': config.threshold = parseFloat(args[++i]); break;
            case '--strategy': case '-s': config.strategy = args[++i]; break;
            case '--exact-cols':
                while (i + 1 < args.length && !args[i + 1].startsWith('-')) {
                    config.exactCols.push(args[++i]);
                }
                break;
            case '--output': case '-o': config.output = args[++i]; break;
        }
    }
    return config;
}

function main() {
    const config = parseArgs();

    let csvPath;
    if (config.input) {
        if (!fs.existsSync(config.input)) {
            console.error(`Error: File '${config.input}' not found.`);
            process.exit(1);
        }
        csvPath = config.input;
    } else {
        console.log('No input file specified. Generating sample dataset...');
        csvPath = generateSampleDataset('sample_data.csv');
    }

    console.log(`Loading data from: ${csvPath}`);
    const csvContent = fs.readFileSync(csvPath, 'utf-8');
    const records = parse(csvContent, { columns: true, skip_empty_lines: true });
    const columns = Object.keys(records[0] || {});
    console.log(`Loaded ${records.length} records with columns: ${columns.join(', ')}`);

    const columnRules = buildColumnRules(columns, config.strategy, config.exactCols);
    console.log(`Using strategy: ${config.strategy}, threshold: ${config.threshold}`);

    console.log('\nFinding exact duplicates...');
    const exactGroups = findExactDuplicates(records);

    console.log('Finding fuzzy duplicates...');
    const fuzzyGroups = findFuzzyDuplicates(records, columnRules, config.threshold, config.strategy);

    printConsoleReport(records, exactGroups, fuzzyGroups);
    generateJsonReport(records, exactGroups, fuzzyGroups, config.output);
}

main();

{
  "name": "duplicate-record-finder-t1",
  "version": "1.0.0",
  "description": "Duplicate Record Finder using fuse.js, csv-parse, and cli-table3",
  "main": "finder.js",
  "scripts": {
    "start": "node finder.js",
    "test": "node finder.js"
  },
  "dependencies": {
    "fuse.js": "7.0.0",
    "csv-parse": "5.5.3",
    "csv-stringify": "6.4.5",
    "cli-table3": "0.6.3"
  }
}

# Duplicate Record Finder - JavaScript Trial 1

Identifies duplicate and near-duplicate records in CSV files using exact matching and fuzzy string matching with configurable similarity thresholds.

## Dependencies
- fuse.js@7.0.0
- csv-parse@5.5.3
- csv-stringify@6.4.5
- cli-table3@0.6.3

## Installation
```bash
npm install
```

## Usage
```bash
# With sample data (auto-generated)
node finder.js

# With custom CSV
node finder.js --input data.csv

# With custom threshold and strategy
node finder.js --input data.csv --threshold 0.85 --strategy jaro-winkler

# With exact match columns
node finder.js --input data.csv --exact-cols email phone

# Custom output path
node finder.js --input data.csv --output results.json
```

## Strategies
- `levenshtein` - Levenshtein distance-based similarity
- `jaro-winkler` - Jaro-Winkler similarity
- `cosine` - Cosine similarity using character bigrams

## Output
- Console report with formatted tables (cli-table3)
- `duplicates_report.json` with detailed findings