Log File Pattern Analyzer (javascript, written by Claude Code)

Claude Code wrote this javascript project from the task below. It installed and ran on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: Log File Pattern Analyzer

Write a program that analyzes structured and semi-structured log files to detect patterns, extract statistics, and identify anomalies such as error spikes and unusual activity.

FUNCTIONAL REQUIREMENTS:
- Accept a log file path as a command-line argument
- Auto-detect common log formats: Apache/Nginx access logs, syslog, and JSON-structured logs
- Parse timestamps, log levels (DEBUG, INFO, WARN, ERROR, FATAL), source identifiers, and message content
- Compute statistics: total entries, entries per log level, entries per hour/day, top 10 most frequent messages (grouped by template after removing variable parts like IPs, timestamps, and IDs)
- Detect error spikes: flag any time window where the error rate exceeds 3x the overall average error rate
- Support filtering by date range via --from and --to flags (ISO 8601 format)
- Support filtering by log level via --level flag (show that level and above)
- Print a summary report to console with counts, top patterns, and detected anomalies
- Save the full analysis as a JSON report file with --output flag (default: log_analysis.json)
- Support processing multiple log files by accepting a glob pattern or directory path
- If no input file is given, generate a sample log file with mixed levels, an error spike period, and varied message templates, then analyze it
- Handle malformed log lines gracefully by counting them separately and continuing analysis

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

#!/usr/bin/env node
"use strict";

/**
 * Log File Pattern Analyzer - Trial 1 (moment + chalk + csv-parse)
 *
 * Analyzes structured/semi-structured log files to detect patterns,
 * extract statistics, and identify anomalies.
 */

const fs = require("fs");
const path = require("path");
const readline = require("readline");
const moment = require("moment");
const chalk = require("chalk");

// ---------------------------------------------------------------------------
// Regex patterns
// ---------------------------------------------------------------------------
const SYSLOG_RE =
  /^(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(\S+)\s+(\S+?):\s+(.*)$/;
const APACHE_RE =
  /^(\S+)\s+\S+\s+\S+\s+\[([^\]]+)\]\s+"(\S+)\s+(\S+)\s+\S+"\s+(\d{3})\s+(\S+)(?:\s+"[^"]*"\s+"[^"]*")?(?:\s+(\d+))?/;

const STATUS_LEVEL = { "2": "INFO", "3": "INFO", "4": "WARNING", "5": "ERROR" };

const LEVEL_KEYWORDS = {
  emerg: "CRITICAL", alert: "CRITICAL", crit: "CRITICAL",
  err: "ERROR", error: "ERROR",
  warn: "WARNING", warning: "WARNING",
  notice: "INFO", info: "INFO",
  debug: "DEBUG",
};

// ---------------------------------------------------------------------------
// Sample log generator
// ---------------------------------------------------------------------------
function generateSampleLog(filepath, numLines = 2000) {
  const levels = ["DEBUG", "INFO", "INFO", "INFO", "WARNING", "ERROR", "CRITICAL"];
  const sources = ["web-server", "auth-service", "db-worker", "scheduler", "cache"];
  const methods = ["GET", "POST", "PUT", "DELETE"];
  const paths = ["/api/users", "/api/orders", "/api/products", "/health", "/login"];
  const messages = [
    "Request processed successfully",
    "Connection established",
    "Cache miss for key user_session",
    "Database query took 320ms",
    "Authentication failed for user admin",
    "Rate limit exceeded",
    "Timeout waiting for upstream",
    "Disk usage above 90%",
    "Memory allocation failed",
    "Service restarted",
  ];

  const baseTime = moment("2024-06-01T00:00:00");
  const lines = [];

  for (let i = 0; i < numLines; i++) {
    const ts = baseTime.clone().add(i * 2 + Math.floor(Math.random() * 4), "seconds");
    const fmtChoice = weightedChoice(["syslog", "apache", "json"], [30, 40, 30]);
    let level;
    if (i >= 800 && i <= 850) {
      level = Math.random() < 0.5 ? "ERROR" : "CRITICAL";
    } else {
      level = levels[Math.floor(Math.random() * levels.length)];
    }

    if (fmtChoice === "syslog") {
      const src = pick(sources);
      const msg = pick(messages);
      const sysTs = ts.format("MMM DD HH:mm:ss");
      const pid = 1000 + Math.floor(Math.random() * 9000);
      lines.push(`${sysTs} ${src} app[${pid}]: [${level}] ${msg}`);
    } else if (fmtChoice === "apache") {
      const ip = `192.168.${1 + Math.floor(Math.random() * 10)}.${1 + Math.floor(Math.random() * 254)}`;
      const method = pick(methods);
      const p = pick(paths);
      const status = { INFO: 200, WARNING: 404, ERROR: 500, DEBUG: 200, CRITICAL: 503 }[level];
      const size = 200 + Math.floor(Math.random() * 50000);
      const rt = 5 + Math.floor(Math.random() * 2000);
      const apacheTs = ts.format("DD/MMM/YYYY:HH:mm:ss +0000");
      lines.push(`${ip} - - [${apacheTs}] "${method} ${p} HTTP/1.1" ${status} ${size} "-" "Mozilla/5.0" ${rt}`);
    } else {
      const record = {
        timestamp: ts.toISOString(),
        level,
        source: pick(sources),
        message: pick(messages),
      };
      lines.push(JSON.stringify(record));
    }

    if (Math.random() < 0.02) {
      lines.push("<<<MALFORMED LINE -- random garbage @#$% >>>");
    }
  }

  fs.writeFileSync(filepath, lines.join("\n") + "\n", "utf8");
  return filepath;
}

function pick(arr) {
  return arr[Math.floor(Math.random() * arr.length)];
}

function weightedChoice(items, weights) {
  const total = weights.reduce((a, b) => a + b, 0);
  let r = Math.random() * total;
  for (let i = 0; i < items.length; i++) {
    r -= weights[i];
    if (r <= 0) return items[i];
  }
  return items[items.length - 1];
}

// ---------------------------------------------------------------------------
// Parsing
// ---------------------------------------------------------------------------
function inferLevel(message) {
  const lower = message.toLowerCase();
  for (const [kw, lvl] of Object.entries(LEVEL_KEYWORDS)) {
    if (lower.includes(kw)) return lvl;
  }
  const m = message.match(/\[(\w+)\]/);
  if (m) {
    const cand = m[1].toUpperCase();
    if (["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"].includes(cand)) return cand;
  }
  return "INFO";
}

function parseLine(line) {
  line = line.trim();
  if (!line) return null;

  // JSON
  if (line.startsWith("{")) {
    try {
      const obj = JSON.parse(line);
      if (obj.timestamp) {
        return {
          timestamp: moment(obj.timestamp),
          level: (obj.level || "INFO").toUpperCase(),
          source: obj.source || "unknown",
          message: obj.message || "",
          responseTime: obj.response_time || null,
          format: "json",
        };
      }
    } catch (_) {}
  }

  // Apache
  let m = line.match(APACHE_RE);
  if (m) {
    const ts = moment(m[2], "DD/MMM/YYYY:HH:mm:ss Z");
    const statusClass = m[5][0];
    const level = STATUS_LEVEL[statusClass] || "INFO";
    const rt = m[7] ? parseInt(m[7], 10) : null;
    return {
      timestamp: ts,
      level,
      source: m[1],
      message: `${m[3]} ${m[4]} ${m[5]}`,
      responseTime: rt,
      format: "apache",
    };
  }

  // Syslog
  m = line.match(SYSLOG_RE);
  if (m) {
    let ts = moment(m[1], "MMM DD HH:mm:ss");
    if (ts.isValid()) {
      ts.year(moment().year());
    }
    const level = inferLevel(m[4]);
    return {
      timestamp: ts,
      level,
      source: m[2],
      message: m[4],
      responseTime: null,
      format: "syslog",
    };
  }

  return null;
}

// ---------------------------------------------------------------------------
// Statistics helpers
// ---------------------------------------------------------------------------
function percentile(sorted, p) {
  if (sorted.length === 0) return 0;
  const idx = (p / 100) * (sorted.length - 1);
  const lower = Math.floor(idx);
  const upper = Math.ceil(idx);
  if (lower === upper) return sorted[lower];
  return sorted[lower] + (sorted[upper] - sorted[lower]) * (idx - lower);
}

function mean(arr) {
  return arr.reduce((a, b) => a + b, 0) / arr.length;
}

function stddev(arr) {
  const m = mean(arr);
  const variance = arr.reduce((sum, v) => sum + (v - m) ** 2, 0) / arr.length;
  return Math.sqrt(variance);
}

// ---------------------------------------------------------------------------
// Core analysis
// ---------------------------------------------------------------------------
async function analyseLogs(filepath) {
  const records = [];
  let malformed = 0;
  let totalLines = 0;

  const rl = readline.createInterface({
    input: fs.createReadStream(filepath, { encoding: "utf8" }),
    crlfDelay: Infinity,
  });

  for await (const line of rl) {
    totalLines++;
    const parsed = parseLine(line);
    if (parsed === null) {
      malformed++;
    } else {
      records.push(parsed);
    }
  }

  if (records.length === 0) {
    return {
      file: filepath,
      total_lines: totalLines,
      parsed_lines: 0,
      malformed_lines: malformed,
      error: "No parseable log lines found.",
    };
  }

  // Sort by timestamp
  records.sort((a, b) => a.timestamp.valueOf() - b.timestamp.valueOf());

  const report = {
    file: filepath,
    total_lines: totalLines,
    parsed_lines: records.length,
    malformed_lines: malformed,
  };

  // Level distribution
  const levelCounts = {};
  for (const r of records) {
    levelCounts[r.level] = (levelCounts[r.level] || 0) + 1;
  }
  report.level_distribution = levelCounts;

  // Error rate
  const errorCount = records.filter(
    (r) => r.level === "ERROR" || r.level === "CRITICAL"
  ).length;
  report.error_count = errorCount;
  report.error_rate = +(errorCount / records.length * 100).toFixed(2);

  // Format distribution
  const fmtCounts = {};
  for (const r of records) {
    fmtCounts[r.format] = (fmtCounts[r.format] || 0) + 1;
  }
  report.format_distribution = fmtCounts;

  // Top sources
  const srcCounts = {};
  for (const r of records) {
    srcCounts[r.source] = (srcCounts[r.source] || 0) + 1;
  }
  const topSources = Object.entries(srcCounts)
    .sort((a, b) => b[1] - a[1])
    .slice(0, 10);
  report.top_sources = Object.fromEntries(topSources);

  // Response time stats
  const rtValues = records
    .filter((r) => r.responseTime !== null)
    .map((r) => r.responseTime);
  if (rtValues.length > 0) {
    const sorted = [...rtValues].sort((a, b) => a - b);
    report.response_time = {
      count: rtValues.length,
      mean_ms: +mean(rtValues).toFixed(2),
      median_ms: +percentile(sorted, 50).toFixed(2),
      p95_ms: +percentile(sorted, 95).toFixed(2),
      p99_ms: +percentile(sorted, 99).toFixed(2),
      max_ms: sorted[sorted.length - 1],
      min_ms: sorted[0],
    };
  }

  // Time window analysis
  const validTs = records.filter((r) => r.timestamp && r.timestamp.isValid());
  if (validTs.length > 1) {
    const tsMin = validTs[0].timestamp;
    const tsMax = validTs[validTs.length - 1].timestamp;
    const durationSec = tsMax.diff(tsMin, "seconds");

    report.time_range = {
      start: tsMin.toISOString(),
      end: tsMax.toISOString(),
      duration_seconds: durationSec,
    };

    // Bucket size
    let windowSec, windowLabel;
    if (durationSec <= 3600) {
      windowSec = 60; windowLabel = "1min";
    } else if (durationSec <= 86400) {
      windowSec = 300; windowLabel = "5min";
    } else {
      windowSec = 3600; windowLabel = "1h";
    }

    // Bucket events
    const allBuckets = {};
    const errorBuckets = {};
    for (const r of validTs) {
      const offset = Math.floor(r.timestamp.diff(tsMin, "seconds") / windowSec);
      const bucketKey = tsMin.clone().add(offset * windowSec, "seconds").toISOString();
      allBuckets[bucketKey] = (allBuckets[bucketKey] || 0) + 1;
      if (r.level === "ERROR" || r.level === "CRITICAL") {
        errorBuckets[bucketKey] = (errorBuckets[bucketKey] || 0) + 1;
      }
    }

    const allCounts = Object.values(allBuckets);
    if (allCounts.length > 0) {
      report.request_rate = {
        bucket: windowLabel,
        mean_per_bucket: +mean(allCounts).toFixed(2),
        max_per_bucket: Math.max(...allCounts),
        min_per_bucket: Math.min(...allCounts),
      };
    }

    // Anomaly detection
    const anomalies = [];
    const bucketKeys = Object.keys(allBuckets).sort();
    const errorValues = bucketKeys.map((k) => errorBuckets[k] || 0);

    if (errorValues.length > 3) {
      const m = mean(errorValues);
      const s = stddev(errorValues);
      if (s > 0) {
        for (let i = 0; i < errorValues.length; i++) {
          const z = (errorValues[i] - m) / s;
          if (z > 2.0) {
            anomalies.push({
              window: bucketKeys[i],
              error_count: errorValues[i],
              z_score: +z.toFixed(2),
              type: "error_spike",
            });
          }
        }
      }
    }

    // Repeated errors
    const errorMsgs = {};
    const errorRecords = records.filter(
      (r) => r.level === "ERROR" || r.level === "CRITICAL"
    );
    for (const r of errorRecords) {
      errorMsgs[r.message] = (errorMsgs[r.message] || 0) + 1;
    }
    const sortedMsgs = Object.entries(errorMsgs).sort((a, b) => b[1] - a[1]);
    for (const [msg, cnt] of sortedMsgs.slice(0, 5)) {
      if (cnt > errorRecords.length * 0.2) {
        anomalies.push({
          type: "repeated_error",
          message: msg,
          count: cnt,
          percentage: +(cnt / errorRecords.length * 100).toFixed(2),
        });
      }
    }

    report.anomalies = anomalies;
    report.anomaly_count = anomalies.length;
  }

  return report;
}

// ---------------------------------------------------------------------------
// Console output
// ---------------------------------------------------------------------------
function printReport(report) {
  const sep = "=".repeat(70);
  console.log("\n" + chalk.blue(sep));
  console.log(chalk.blue.bold("  LOG FILE PATTERN ANALYZER - ANALYSIS REPORT"));
  console.log(chalk.blue(sep));
  console.log(`  File:            ${report.file}`);
  console.log(`  Total lines:     ${report.total_lines}`);
  console.log(`  Parsed lines:    ${report.parsed_lines}`);
  console.log(`  Malformed lines: ${report.malformed_lines}`);
  console.log();

  if (report.error) {
    console.log(chalk.red.bold(`  ERROR: ${report.error}`));
    return;
  }

  console.log(chalk.cyan("  -- Level Distribution --"));
  const levelColors = {
    DEBUG: chalk.gray, INFO: chalk.blue, WARNING: chalk.yellow,
    ERROR: chalk.red, CRITICAL: chalk.bgRed.white,
  };
  for (const [level, count] of Object.entries(report.level_distribution || {}).sort()) {
    const pct = (count / report.parsed_lines * 100).toFixed(1);
    const bar = "#".repeat(Math.floor(pct / 2));
    const colorFn = levelColors[level] || chalk.white;
    console.log(`    ${colorFn(level.padEnd(10))} ${String(count).padStart(6)}  (${pct.padStart(5)}%)  ${bar}`);
  }

  console.log();
  const errStyle = report.error_rate > 10 ? chalk.red.bold : report.error_rate > 5 ? chalk.yellow : chalk.green;
  console.log(`  Error count: ${chalk.bold(report.error_count)}`);
  console.log(`  Error rate:  ${errStyle(report.error_rate + "%")}`);
  console.log();

  if (report.response_time) {
    const rt = report.response_time;
    console.log(chalk.cyan("  -- Response Time (ms) --"));
    console.log(`    Mean:   ${rt.mean_ms}`);
    console.log(`    Median: ${rt.median_ms}`);
    console.log(`    P95:    ${rt.p95_ms}`);
    console.log(`    P99:    ${rt.p99_ms}`);
    console.log(`    Max:    ${rt.max_ms}`);
    console.log();
  }

  if (report.time_range) {
    const tr = report.time_range;
    console.log(chalk.cyan("  -- Time Range --"));
    console.log(`    Start:    ${tr.start}`);
    console.log(`    End:      ${tr.end}`);
    console.log(`    Duration: ${tr.duration_seconds}s`);
    console.log();
  }

  if (report.request_rate) {
    const rr = report.request_rate;
    console.log(chalk.cyan(`  -- Request Rate (${rr.bucket} buckets) --`));
    console.log(`    Mean: ${rr.mean_per_bucket}`);
    console.log(`    Max:  ${rr.max_per_bucket}`);
    console.log(`    Min:  ${rr.min_per_bucket}`);
    console.log();
  }

  if (report.format_distribution) {
    console.log(chalk.cyan("  -- Log Format Distribution --"));
    for (const [fmt, cnt] of Object.entries(report.format_distribution)) {
      console.log(`    ${fmt.padEnd(10)} ${String(cnt).padStart(6)}`);
    }
    console.log();
  }

  if (report.top_sources) {
    console.log(chalk.cyan("  -- Top Sources --"));
    for (const [src, cnt] of Object.entries(report.top_sources)) {
      console.log(`    ${src.padEnd(25)} ${String(cnt).padStart(6)}`);
    }
    console.log();
  }

  const anomalies = report.anomalies || [];
  console.log(chalk.cyan(`  -- Anomalies Detected: ${anomalies.length} --`));
  for (let i = 0; i < anomalies.length; i++) {
    const a = anomalies[i];
    if (a.type === "error_spike") {
      console.log(chalk.red(`    [${i + 1}] ERROR SPIKE at ${a.window} (count=${a.error_count}, z=${a.z_score})`));
    } else if (a.type === "repeated_error") {
      console.log(chalk.yellow(`    [${i + 1}] REPEATED ERROR: "${a.message}" (count=${a.count}, ${a.percentage}%)`));
    } else {
      console.log(`    [${i + 1}] ${JSON.stringify(a)}`);
    }
  }

  console.log(chalk.blue(sep));
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
  let logfile = process.argv[2] || null;
  const outputArg = process.argv.indexOf("-o");
  const outputPath =
    outputArg !== -1 && process.argv[outputArg + 1]
      ? process.argv[outputArg + 1]
      : "analysis_report.json";

  if (!logfile) {
    const samplePath = path.join(process.cwd(), "sample.log");
    console.log(
      chalk.yellow(`No log file specified. Generating sample log at ${samplePath} ...`)
    );
    generateSampleLog(samplePath);
    logfile = samplePath;
  }

  if (!fs.existsSync(logfile)) {
    console.error(chalk.red(`Error: File not found: ${logfile}`));
    process.exit(1);
  }

  console.log(`Analyzing ${logfile} ...`);
  const report = await analyseLogs(logfile);
  printReport(report);

  fs.writeFileSync(outputPath, JSON.stringify(report, null, 2), "utf8");
  console.log(chalk.green(`\nJSON report written to ${outputPath}`));
}

main().catch((err) => {
  console.error(chalk.red(`Fatal error: ${err.message}`));
  process.exit(1);
});

{
  "name": "log-pattern-analyzer-trial1",
  "version": "1.0.0",
  "description": "Log File Pattern Analyzer using moment + chalk + csv-parse",
  "main": "analyzer.js",
  "scripts": {
    "start": "node analyzer.js"
  },
  "dependencies": {
    "moment": "2.30.1",
    "chalk": "4.1.2",
    "csv-parse": "5.5.6"
  }
}

# Log File Pattern Analyzer (Trial 1 - JavaScript: moment + chalk + csv-parse)

Analyzes structured/semi-structured log files to detect patterns, extract statistics, and identify anomalies.

## Dependencies
- moment 2.30.1
- chalk 4.1.2
- csv-parse 5.5.6

## Installation
```bash
npm install
```

## Usage
```bash
# Analyze a log file
node analyzer.js /path/to/logfile.log

# Generate sample log and analyze
node analyzer.js

# Custom output path
node analyzer.js /path/to/logfile.log -o my_report.json
```

## Features
- Parses syslog, Apache combined, and JSON log formats
- Extracts timestamps, log levels, sources
- Computes error rates, request rates, response time percentiles
- Detects anomalies via z-score method
- Time-window bucketing analysis
- Colored console output via chalk
- JSON report output
- Handles malformed lines gracefully