Log File Pattern Analyzer (javascript, written by Codex)

Codex wrote this javascript project from the task below. It installed and ran on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: Log File Pattern Analyzer

Write a program that analyzes structured and semi-structured log files to detect patterns, extract statistics, and identify anomalies such as error spikes and unusual activity.

FUNCTIONAL REQUIREMENTS:
- Accept a log file path as a command-line argument
- Auto-detect common log formats: Apache/Nginx access logs, syslog, and JSON-structured logs
- Parse timestamps, log levels (DEBUG, INFO, WARN, ERROR, FATAL), source identifiers, and message content
- Compute statistics: total entries, entries per log level, entries per hour/day, top 10 most frequent messages (grouped by template after removing variable parts like IPs, timestamps, and IDs)
- Detect error spikes: flag any time window where the error rate exceeds 3x the overall average error rate
- Support filtering by date range via --from and --to flags (ISO 8601 format)
- Support filtering by log level via --level flag (show that level and above)
- Print a summary report to console with counts, top patterns, and detected anomalies
- Save the full analysis as a JSON report file with --output flag (default: log_analysis.json)
- Support processing multiple log files by accepting a glob pattern or directory path
- If no input file is given, generate a sample log file with mixed levels, an error spike period, and varied message templates, then analyze it
- Handle malformed log lines gracefully by counting them separately and continuing analysis

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

{
  "metadata": {
    "analyzedAt": "2026-03-02T18:19:02.719Z",
    "files": [
      "D:\\LLM Dependency Manager\\LLM-Dependency-Manager\\TMLR\\code_generation\\codex_generated\\p_05\\javascript\\sample.log"
    ],
    "from": null,
    "to": null,
    "levelThreshold": "DEBUG"
  },
  "totals": {
    "parsedEntries": 302,
    "malformedLines": 1,
    "timestampedEntries": 302
  },
  "entriesPerLevel": {
    "DEBUG": 0,
    "INFO": 235,
    "WARN": 7,
    "ERROR": 60,
    "FATAL": 0
  },
  "entriesPerHour": {
    "2001-01-10T20:00:00Z": 1,
    "2026-01-10T08:00:00Z": 60,
    "2026-01-10T09:00:00Z": 60,
    "2026-01-10T10:00:00Z": 60,
    "2026-01-10T11:00:00Z": 60,
    "2026-01-10T12:00:00Z": 60,
    "2026-01-10T13:00:00Z": 1
  },
  "entriesPerDay": {
    "2001-01-10": 1,
    "2026-01-10": 301
  },
  "topMessagePatterns": [
    {
      "template": "<ID> completed <ID> <ID>",
      "count": 240
    },
    {
      "template": "Payment failure for <ID>=<NUM> ip=<IP>",
      "count": 60
    },
    {
      "template": "GET /health HTTP/<NUM>.<NUM> status=<NUM> bytes=<NUM>",
      "count": 1
    },
    {
      "template": "WARN job <ID> delayed by 45s",
      "count": 1
    }
  ],
  "anomalies": {
    "errorSpikes": [
      {
        "windowStart": "2026-01-10T12:00:00Z",
        "totalEntries": 60,
        "errorEntries": 60,
        "errorRate": 1,
        "averageErrorRate": 0.1986754966887417
      }
    ]
  }
}

{
  "name": "log-file-pattern-analyzer",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "log-file-pattern-analyzer",
      "version": "1.0.0"
    }
  }
}

{
  "name": "log-file-pattern-analyzer",
  "version": "1.0.0",
  "description": "Analyze structured and semi-structured logs for patterns and anomalies.",
  "type": "module",
  "main": "src/index.js",
  "scripts": {
    "start": "node src/index.js",
    "analyze": "node src/index.js"
  },
  "engines": {
    "node": ">=20.0.0"
  },
  "dependencies": {},
  "devDependencies": {}
}

# Log File Pattern Analyzer (JavaScript)

Analyzes Apache/Nginx, syslog, JSON logs, and generic level logs to compute statistics, find templates, and detect error spikes.

## Features

- Input: file path, directory path, or wildcard pattern.
- Auto-detects log formats:
  - Apache/Nginx access logs
  - syslog
  - JSON structured logs
  - generic level logs
- Parses timestamp, level, source, message.
- Stats:
  - total entries
  - malformed lines
  - entries per level
  - entries per hour/day
  - top 10 message templates
- Error spike detection:
  - flags windows where error rate > 3x overall average.
- Filters:
  - `--from` / `--to` (ISO 8601)
  - `--level` (and above)
- Console summary + JSON report output (`--output`, default `log_analysis.json`).
- No input: generates sample log with error spike and mixed templates.

## Requirements

- Ubuntu 22.04
- Node.js 20+

## Dependencies (Pinned)

No external dependencies.

- Direct dependencies: none
- Transitive dependencies: none

## Run

```bash
cd TMLR/code_generation/codex_generated/p_05/javascript
node src/index.js ./app.log
```

Directory:

```bash
node src/index.js ./logs
```

Wildcard:

```bash
node src/index.js "./logs/*.log"
```

Filters and output:

```bash
node src/index.js ./logs --from 2026-01-10T00:00:00Z --to 2026-01-11T00:00:00Z --level WARN --output report.json
```

No args:

```bash
node src/index.js
```

#!/usr/bin/env node
import fs from "node:fs/promises";
import path from "node:path";

const LEVELS = ["DEBUG", "INFO", "WARN", "ERROR", "FATAL"];
const LEVEL_RANK = new Map(LEVELS.map((l, i) => [l, i]));

function levelAtLeast(level, threshold) {
  return LEVEL_RANK.get(level) >= LEVEL_RANK.get(threshold);
}

function normalizeLevel(raw) {
  if (!raw) return "INFO";
  const u = String(raw).toUpperCase();
  if (u === "WARNING") return "WARN";
  if (u === "ERR") return "ERROR";
  if (u === "CRITICAL") return "FATAL";
  if (LEVEL_RANK.has(u)) return u;
  return null;
}

function parseTimestamp(text) {
  if (!text) return null;
  const d = new Date(text);
  if (!Number.isNaN(d.getTime())) return d;

  const syslog = text.match(/^([A-Z][a-z]{2}\s+\d+\s+\d\d:\d\d:\d\d)$/);
  if (syslog) {
    const year = new Date().getUTCFullYear();
    const d2 = new Date(`${syslog[1]} ${year} UTC`);
    if (!Number.isNaN(d2.getTime())) return d2;
  }
  return null;
}

function templateize(message) {
  let out = message;
  out = out.replace(/\b\d{4}-\d{2}-\d{2}[T ][\d:.+-Z]+\b/g, "<TIMESTAMP>");
  out = out.replace(/\b\d{1,3}(?:\.\d{1,3}){3}\b/g, "<IP>");
  out = out.replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, "<UUID>");
  out = out.replace(/\b0x[0-9a-f]+\b/gi, "<HEX>");
  out = out.replace(/\b(id|user|session|req|trace)[=:]?[A-Za-z0-9_-]+\b/gi, "<ID>");
  out = out.replace(/\b\d+\b/g, "<NUM>");
  out = out.replace(/\s+/g, " ").trim();
  return out;
}

function parseJsonLog(line) {
  if (!line.trim().startsWith("{")) return null;
  try {
    const obj = JSON.parse(line);
    const msg = obj.message ?? obj.msg ?? obj.event ?? "";
    const level = normalizeLevel(obj.level ?? obj.severity ?? obj.log_level ?? "INFO");
    if (!level) return null;
    const tsRaw = obj.timestamp ?? obj.time ?? obj.datetime ?? obj.date ?? null;
    return {
      format: "json",
      timestamp: parseTimestamp(tsRaw),
      level,
      source: obj.source ?? obj.logger ?? obj.service ?? null,
      message: String(msg),
    };
  } catch {
    return null;
  }
}

const APACHE_RE =
  /^(\d{1,3}(?:\.\d{1,3}){3})\s+\S+\s+\S+\s+\[([^\]]+)\]\s+"([^"]*)"\s+(\d{3})\s+(\S+)(?:\s+"([^"]*)"\s+"([^"]*)")?/;

function parseApache(line) {
  const m = line.match(APACHE_RE);
  if (!m) return null;
  const ts = parseTimestamp(m[2].replace(":", " ").replace(/\//g, " "));
  const status = Number(m[4]);
  const level = status >= 500 ? "ERROR" : status >= 400 ? "WARN" : "INFO";
  return {
    format: "apache",
    timestamp: ts,
    level,
    source: m[1],
    message: `${m[3]} status=${m[4]} bytes=${m[5]}`,
  };
}

const SYSLOG_RE = /^([A-Z][a-z]{2}\s+\d+\s+\d\d:\d\d:\d\d)\s+(\S+)\s+([^:]+):\s*(.*)$/;

function parseSyslog(line) {
  const m = line.match(SYSLOG_RE);
  if (!m) return null;
  const ts = parseTimestamp(m[1]);
  const msg = m[4];
  let level = "INFO";
  const lvlMatch = msg.match(/\b(DEBUG|INFO|WARN|WARNING|ERROR|FATAL|CRITICAL)\b/i);
  if (lvlMatch) {
    level = normalizeLevel(lvlMatch[1]) ?? "INFO";
  }
  return {
    format: "syslog",
    timestamp: ts,
    level,
    source: m[3],
    message: msg,
  };
}

function parseGeneric(line) {
  const m = line.match(
    /^(\d{4}-\d{2}-\d{2}[T ][\d:.+-Z]+)?\s*\[?(DEBUG|INFO|WARN|WARNING|ERROR|FATAL|CRITICAL)\]?\s*([A-Za-z0-9_.-]+)?\s*[-:]?\s*(.*)$/i,
  );
  if (!m) return null;
  const level = normalizeLevel(m[2]);
  if (!level) return null;
  return {
    format: "generic",
    timestamp: parseTimestamp(m[1]),
    level,
    source: m[3] || null,
    message: m[4] || line,
  };
}

function parseLine(line) {
  return parseJsonLog(line) ?? parseApache(line) ?? parseSyslog(line) ?? parseGeneric(line);
}

function wildcardToRegex(pattern) {
  const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
  return new RegExp(`^${escaped.replace(/\*/g, ".*").replace(/\?/g, ".")}$`, "i");
}

async function resolveInputs(inputArg) {
  if (!inputArg) return [];
  const abs = path.resolve(process.cwd(), inputArg);
  const hasWildcard = /[*?]/.test(inputArg);

  if (hasWildcard) {
    const dir = path.resolve(process.cwd(), path.dirname(inputArg));
    const basePattern = path.basename(inputArg);
    const re = wildcardToRegex(basePattern);
    const entries = await fs.readdir(dir, { withFileTypes: true });
    return entries
      .filter((e) => e.isFile() && re.test(e.name))
      .map((e) => path.join(dir, e.name))
      .sort();
  }

  try {
    const st = await fs.stat(abs);
    if (st.isDirectory()) {
      const entries = await fs.readdir(abs, { withFileTypes: true });
      return entries
        .filter((e) => e.isFile() && /\.(log|txt|jsonl?)$/i.test(e.name))
        .map((e) => path.join(abs, e.name))
        .sort();
    }
    return [abs];
  } catch {
    return [];
  }
}

function createSampleLogs() {
  const lines = [];
  const start = new Date("2026-01-10T08:00:00Z");
  for (let i = 0; i < 240; i += 1) {
    const t = new Date(start.getTime() + i * 60_000);
    const iso = t.toISOString();
    const level = i % 40 === 0 ? "WARN" : "INFO";
    lines.push(`${iso} [${level}] api-gateway - Request completed id=req-${1000 + i} user=u${i % 20}`);
  }

  for (let i = 0; i < 60; i += 1) {
    const t = new Date(new Date("2026-01-10T12:00:00Z").getTime() + i * 30_000);
    const iso = t.toISOString();
    lines.push(
      JSON.stringify({
        timestamp: iso,
        level: "ERROR",
        source: "payment-service",
        message: `Payment failure for user_id=${5000 + i} ip=10.0.0.${i % 10}`,
      }),
    );
  }

  lines.push('127.0.0.1 - - [10/Jan/2026:13:10:01 +0000] "GET /health HTTP/1.1" 200 64');
  lines.push('Jan 10 14:00:20 host1 scheduler: WARN job id=abc123 delayed by 45s');
  lines.push("BROKEN LINE WITHOUT FORMAT");

  return `${lines.join("\n")}\n`;
}

function hourBucket(date) {
  return date.toISOString().slice(0, 13) + ":00:00Z";
}

function dayBucket(date) {
  return date.toISOString().slice(0, 10);
}

function computeReport(entries, malformedCount, files, options) {
  const levelCounts = Object.fromEntries(LEVELS.map((l) => [l, 0]));
  const hourly = {};
  const daily = {};
  const templates = {};
  const hourlyStats = {};

  let errorCount = 0;
  let timestampedEntries = 0;

  for (const e of entries) {
    levelCounts[e.level] = (levelCounts[e.level] ?? 0) + 1;
    const tpl = templateize(e.message);
    templates[tpl] = (templates[tpl] ?? 0) + 1;

    if (e.timestamp) {
      timestampedEntries += 1;
      const hb = hourBucket(e.timestamp);
      const db = dayBucket(e.timestamp);
      hourly[hb] = (hourly[hb] ?? 0) + 1;
      daily[db] = (daily[db] ?? 0) + 1;
      if (!hourlyStats[hb]) hourlyStats[hb] = { total: 0, errors: 0 };
      hourlyStats[hb].total += 1;
      if (e.level === "ERROR" || e.level === "FATAL") hourlyStats[hb].errors += 1;
    }
    if (e.level === "ERROR" || e.level === "FATAL") errorCount += 1;
  }

  const avgErrorRate = entries.length > 0 ? errorCount / entries.length : 0;
  const spikes = [];
  for (const [windowStart, stat] of Object.entries(hourlyStats)) {
    const rate = stat.total > 0 ? stat.errors / stat.total : 0;
    if (avgErrorRate > 0 && rate > avgErrorRate * 3) {
      spikes.push({
        windowStart,
        totalEntries: stat.total,
        errorEntries: stat.errors,
        errorRate: rate,
        averageErrorRate: avgErrorRate,
      });
    }
  }
  spikes.sort((a, b) => a.windowStart.localeCompare(b.windowStart));

  const topPatterns = Object.entries(templates)
    .map(([template, count]) => ({ template, count }))
    .sort((a, b) => (b.count - a.count) || a.template.localeCompare(b.template))
    .slice(0, 10);

  return {
    metadata: {
      analyzedAt: new Date().toISOString(),
      files,
      from: options.from ? options.from.toISOString() : null,
      to: options.to ? options.to.toISOString() : null,
      levelThreshold: options.level,
    },
    totals: {
      parsedEntries: entries.length,
      malformedLines: malformedCount,
      timestampedEntries,
    },
    entriesPerLevel: levelCounts,
    entriesPerHour: Object.fromEntries(Object.entries(hourly).sort(([a], [b]) => a.localeCompare(b))),
    entriesPerDay: Object.fromEntries(Object.entries(daily).sort(([a], [b]) => a.localeCompare(b))),
    topMessagePatterns: topPatterns,
    anomalies: {
      errorSpikes: spikes,
    },
  };
}

function printSummary(report) {
  console.log("Log File Pattern Analyzer");
  console.log("=========================");
  console.log(`Files analyzed : ${report.metadata.files.length}`);
  console.log(`Parsed entries : ${report.totals.parsedEntries}`);
  console.log(`Malformed lines: ${report.totals.malformedLines}`);
  console.log(`Level filter   : ${report.metadata.levelThreshold}`);
  if (report.metadata.from || report.metadata.to) {
    console.log(`Date range     : ${report.metadata.from ?? "-"} to ${report.metadata.to ?? "-"}`);
  }

  console.log("\nEntries per level:");
  for (const lvl of LEVELS) console.log(`- ${lvl}: ${report.entriesPerLevel[lvl] ?? 0}`);

  console.log("\nTop patterns:");
  if (report.topMessagePatterns.length === 0) {
    console.log("- (none)");
  } else {
    report.topMessagePatterns.forEach((p, i) => {
      console.log(`${i + 1}. ${p.count}x  ${p.template}`);
    });
  }

  console.log("\nDetected anomalies:");
  if (report.anomalies.errorSpikes.length === 0) {
    console.log("- No error spikes detected.");
  } else {
    for (const s of report.anomalies.errorSpikes) {
      console.log(
        `- ${s.windowStart}: error_rate=${s.errorRate.toFixed(3)} avg=${s.averageErrorRate.toFixed(3)} (${s.errorEntries}/${s.totalEntries})`,
      );
    }
  }
}

async function run() {
  const args = process.argv.slice(2);
  let from = null;
  let to = null;
  let level = "DEBUG";
  let output = "log_analysis.json";
  let inputArg = null;

  for (let i = 0; i < args.length; i += 1) {
    const arg = args[i];
    if (arg === "--from") {
      i += 1;
      from = new Date(args[i]);
    } else if (arg === "--to") {
      i += 1;
      to = new Date(args[i]);
    } else if (arg === "--level") {
      i += 1;
      level = normalizeLevel(args[i] ?? "");
    } else if (arg === "--output") {
      i += 1;
      output = args[i] ?? output;
    } else {
      inputArg = arg;
    }
  }

  if (!LEVEL_RANK.has(level)) {
    console.error("Invalid --level. Use DEBUG|INFO|WARN|ERROR|FATAL");
    process.exitCode = 1;
    return;
  }
  if (from && Number.isNaN(from.getTime())) {
    console.error("Invalid --from timestamp. Use ISO 8601.");
    process.exitCode = 1;
    return;
  }
  if (to && Number.isNaN(to.getTime())) {
    console.error("Invalid --to timestamp. Use ISO 8601.");
    process.exitCode = 1;
    return;
  }

  let files = [];
  if (!inputArg) {
    const sample = path.resolve(process.cwd(), "sample.log");
    await fs.writeFile(sample, createSampleLogs(), "utf8");
    files = [sample];
    console.log(`No input provided. Generated sample log: ${sample}`);
  } else {
    files = await resolveInputs(inputArg);
    if (files.length === 0) {
      console.error("No input log files matched.");
      process.exitCode = 1;
      return;
    }
  }

  const entries = [];
  let malformedLines = 0;

  for (const file of files) {
    let text;
    try {
      text = await fs.readFile(file, "utf8");
    } catch (error) {
      console.error(`Failed to read ${file}: ${error.message}`);
      process.exitCode = 1;
      return;
    }

    const lines = text.split(/\r?\n/);
    for (const line of lines) {
      if (line.trim() === "") continue;
      const parsed = parseLine(line);
      if (!parsed) {
        malformedLines += 1;
        continue;
      }

      if (!levelAtLeast(parsed.level, level)) continue;
      if (from && parsed.timestamp && parsed.timestamp < from) continue;
      if (to && parsed.timestamp && parsed.timestamp > to) continue;
      entries.push(parsed);
    }
  }

  const report = computeReport(entries, malformedLines, files, { from, to, level });
  printSummary(report);

  const outPath = path.resolve(process.cwd(), output);
  await fs.writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
  console.log(`\nSaved JSON report: ${outPath}`);
}

run().catch((error) => {
  console.error(error.message);
  process.exitCode = 1;
});