Structured Log Processor (cpp, written by Codex)

Codex wrote this cpp project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: Structured Log Processor

Write a program that parses, queries, transforms, and aggregates structured log data in JSON Lines format, supporting filtering, field extraction, statistical aggregation, and output formatting.

FUNCTIONAL REQUIREMENTS:
- Accept a log file path as a command-line argument (JSON Lines format: one JSON object per line)
- Support filtering log entries via --where flag with field comparisons (e.g., --where "level==ERROR" or --where "response_time>500" or --where "status!=200")
- Support multiple filters combined with AND logic; support OR logic via --or flag
- Support field selection via --fields flag (comma-separated list of field names to include in output)
- Support aggregation operations via --group-by and --aggregate flags: count, sum, avg, min, max, and percentile(N) grouped by a specified field (e.g., --group-by status --aggregate "count,avg:response_time")
- Support time-based aggregation: group by time windows (--time-window flag: 1m, 5m, 1h, 1d) on a specified timestamp field (--time-field flag)
- Support sorting via --sort flag (field name with optional :asc or :desc suffix)
- Support limiting output via --limit flag and skipping via --offset flag
- Support output in multiple formats via --format flag: json (default), csv, table (formatted console table), and jsonl (JSON Lines)
- Compute and display summary statistics for numeric fields: count, min, max, mean, median, p95, p99
- Support extracting unique values of a field via --distinct flag
- Print results to console by default
- Save results to a file via --output flag
- If no input file is given, generate a sample web server access log with 1000 entries containing fields (timestamp, method, path, status, response_time, user_agent, ip), then demonstrate: filtering ERROR entries, computing average response time grouped by HTTP method, finding the top 10 slowest requests, and computing hourly request counts
- Handle errors: malformed JSON lines (skip with warning and count), missing fields in filter expressions, type mismatches in comparisons, and very large files

Create a complete C++ project for a clean Ubuntu 22.04 machine with only G++ 12+ and CMake 3.22+ installed. Include:
- Source code
- CMakeLists.txt with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

cmake_minimum_required(VERSION 3.14)
project(StructuredLogProcessor VERSION 1.0 LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(nlohmann_json REQUIRED)
find_package(CLI11 REQUIRED)

add_executable(logproc src/main.cpp)
target_link_libraries(logproc PRIVATE nlohmann_json::nlohmann_json CLI11::CLI11)

install(TARGETS logproc DESTINATION bin)

# Structured Log Processor (C++ - Trial 1)

## Description
Parses, queries, and aggregates JSON Lines log files with support for filtering,
field selection, and time-based aggregation. Uses nlohmann/json for JSON parsing
and CLI11 for command-line interface.

## Dependencies
- **nlohmann-json**: Header-only JSON library for modern C++
- **CLI11**: Header-only CLI parsing library with subcommand support

## Build

```bash
mkdir build && cd build
cmake ..
make
```

## Usage

### Query logs with filters
```bash
./logproc query access.jsonl -f "level==ERROR" -s "timestamp,message" --pretty
```

### Count by field
```bash
./logproc count-by access.jsonl level
```

### Numeric statistics
```bash
./logproc stats access.jsonl response_time
```

### Time series aggregation
```bash
./logproc timeseries access.jsonl timestamp -i hour
```

### Top N values
```bash
./logproc top access.jsonl status_code -n 5
```

## Input Format
Expects JSON Lines format (one JSON object per line):
```json
{"timestamp": "2024-01-15T10:30:00Z", "level": "INFO", "message": "Request processed", "response_time": 42}
```

/**
 * Structured Log Processor
 * Parses/queries/aggregates JSON Lines logs with filtering, field selection,
 * and time-based aggregation.
 * Uses nlohmann/json for JSON parsing and CLI11 for command-line interface.
 */

#include <nlohmann/json.hpp>
#include <CLI/CLI.hpp>

#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <map>
#include <algorithm>
#include <numeric>
#include <sstream>
#include <iomanip>
#include <ctime>
#include <regex>

using json = nlohmann::json;

/**
 * Get a nested field from a JSON object using dot notation.
 */
json get_nested_field(const json& obj, const std::string& field_path) {
    std::istringstream iss(field_path);
    std::string part;
    json current = obj;
    while (std::getline(iss, part, '.')) {
        if (!current.is_object() || !current.contains(part)) {
            return nullptr;
        }
        current = current[part];
    }
    return current;
}

/**
 * Convert a JSON value to string representation.
 */
std::string json_to_string(const json& val) {
    if (val.is_string()) return val.get<std::string>();
    if (val.is_number_integer()) return std::to_string(val.get<int64_t>());
    if (val.is_number_float()) return std::to_string(val.get<double>());
    if (val.is_boolean()) return val.get<bool>() ? "true" : "false";
    if (val.is_null()) return "<null>";
    return val.dump();
}

struct FilterExpression {
    std::string field;
    std::string op;
    std::string value;
};

/**
 * Parse a filter expression string like "level==ERROR".
 */
FilterExpression parse_filter(const std::string& expr) {
    std::vector<std::string> operators = {"!=", ">=", "<=", "==", ">", "<", "contains", "startswith", "endswith"};
    for (const auto& op : operators) {
        auto pos = expr.find(op);
        if (pos != std::string::npos && pos > 0) {
            FilterExpression f;
            f.field = expr.substr(0, pos);
            f.op = op;
            f.value = expr.substr(pos + op.length());
            // trim whitespace
            f.field.erase(0, f.field.find_first_not_of(' '));
            f.field.erase(f.field.find_last_not_of(' ') + 1);
            f.value.erase(0, f.value.find_first_not_of(' '));
            f.value.erase(f.value.find_last_not_of(' ') + 1);
            return f;
        }
    }
    throw std::runtime_error("Invalid filter expression: " + expr);
}

/**
 * Check if a record matches a filter condition.
 */
bool matches_filter(const json& record, const FilterExpression& filter) {
    json field_val = get_nested_field(record, filter.field);
    if (field_val.is_null()) return false;
    std::string record_str = json_to_string(field_val);

    if (filter.op == "==") return record_str == filter.value;
    if (filter.op == "!=") return record_str != filter.value;
    if (filter.op == "contains") {
        std::string lower_rec = record_str, lower_val = filter.value;
        std::transform(lower_rec.begin(), lower_rec.end(), lower_rec.begin(), ::tolower);
        std::transform(lower_val.begin(), lower_val.end(), lower_val.begin(), ::tolower);
        return lower_rec.find(lower_val) != std::string::npos;
    }
    if (filter.op == "startswith") return record_str.rfind(filter.value, 0) == 0;
    if (filter.op == "endswith") {
        if (filter.value.size() > record_str.size()) return false;
        return record_str.compare(record_str.size() - filter.value.size(), filter.value.size(), filter.value) == 0;
    }
    // Numeric comparisons
    try {
        double a = std::stod(record_str);
        double b = std::stod(filter.value);
        if (filter.op == ">") return a > b;
        if (filter.op == ">=") return a >= b;
        if (filter.op == "<") return a < b;
        if (filter.op == "<=") return a <= b;
    } catch (...) {
        if (filter.op == ">") return record_str > filter.value;
        if (filter.op == ">=") return record_str >= filter.value;
        if (filter.op == "<") return record_str < filter.value;
        if (filter.op == "<=") return record_str <= filter.value;
    }
    return true;
}

/**
 * Read a JSONL file and return filtered records.
 */
std::vector<json> read_and_filter(const std::string& filepath,
                                   const std::vector<std::string>& filters) {
    std::vector<json> results;
    std::vector<FilterExpression> parsed_filters;
    for (const auto& f : filters) {
        parsed_filters.push_back(parse_filter(f));
    }

    std::ifstream infile(filepath);
    if (!infile.is_open()) {
        std::cerr << "Error: Cannot open file " << filepath << std::endl;
        return results;
    }

    std::string line;
    while (std::getline(infile, line)) {
        if (line.empty()) continue;
        try {
            json record = json::parse(line);
            bool all_match = true;
            for (const auto& f : parsed_filters) {
                if (!matches_filter(record, f)) { all_match = false; break; }
            }
            if (all_match) results.push_back(record);
        } catch (const json::parse_error&) {
            // skip malformed lines
        }
    }
    return results;
}

/**
 * Select specific fields from a record.
 */
json select_fields(const json& record, const std::vector<std::string>& fields) {
    json result = json::object();
    for (const auto& f : fields) {
        json val = get_nested_field(record, f);
        if (!val.is_null()) result[f] = val;
    }
    return result;
}

/**
 * Parse comma-separated string into vector.
 */
std::vector<std::string> split_csv(const std::string& s) {
    std::vector<std::string> parts;
    std::istringstream iss(s);
    std::string part;
    while (std::getline(iss, part, ',')) {
        part.erase(0, part.find_first_not_of(' '));
        part.erase(part.find_last_not_of(' ') + 1);
        if (!part.empty()) parts.push_back(part);
    }
    return parts;
}

/**
 * Format a timestamp string into a time bucket.
 */
std::string format_time_bucket(const std::string& time_str, const std::string& interval) {
    std::tm tm = {};
    std::istringstream ss(time_str);
    // Try ISO 8601 format
    ss >> std::get_time(&tm, "%Y-%m-%dT%H:%M:%S");
    if (ss.fail()) {
        ss.clear();
        ss.str(time_str);
        ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S");
        if (ss.fail()) return "";
    }

    char buf[64];
    if (interval == "minute") std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M", &tm);
    else if (interval == "hour") std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:00", &tm);
    else if (interval == "day") std::strftime(buf, sizeof(buf), "%Y-%m-%d", &tm);
    else if (interval == "month") std::strftime(buf, sizeof(buf), "%Y-%m", &tm);
    else std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:00", &tm);
    return std::string(buf);
}

int main(int argc, char** argv) {
    CLI::App app{"Structured Log Processor - Query and analyze JSON Lines logs"};
    app.require_subcommand(1);

    // Query subcommand
    auto query_cmd = app.add_subcommand("query", "Query log records with filtering and field selection");
    std::string q_logfile, q_fields_str;
    std::vector<std::string> q_filters;
    int q_limit = 0;
    bool q_pretty = false;
    query_cmd->add_option("logfile", q_logfile, "Path to JSON Lines log file")->required();
    query_cmd->add_option("-f,--filter", q_filters, "Filter expressions");
    query_cmd->add_option("-s,--fields", q_fields_str, "Comma-separated field list");
    query_cmd->add_option("-l,--limit", q_limit, "Max records to return");
    query_cmd->add_flag("--pretty", q_pretty, "Pretty-print output");

    // Count-by subcommand
    auto countby_cmd = app.add_subcommand("count-by", "Count records grouped by a field");
    std::string cb_logfile, cb_field;
    std::vector<std::string> cb_filters;
    countby_cmd->add_option("logfile", cb_logfile, "Path to JSON Lines log file")->required();
    countby_cmd->add_option("field", cb_field, "Field to group by")->required();
    countby_cmd->add_option("-f,--filter", cb_filters, "Filter expressions");

    // Stats subcommand
    auto stats_cmd = app.add_subcommand("stats", "Compute numeric statistics for a field");
    std::string st_logfile, st_field;
    std::vector<std::string> st_filters;
    stats_cmd->add_option("logfile", st_logfile, "Path to JSON Lines log file")->required();
    stats_cmd->add_option("field", st_field, "Numeric field to analyze")->required();
    stats_cmd->add_option("-f,--filter", st_filters, "Filter expressions");

    // Timeseries subcommand
    auto ts_cmd = app.add_subcommand("timeseries", "Aggregate records by time intervals");
    std::string ts_logfile, ts_time_field, ts_interval = "hour";
    std::vector<std::string> ts_filters;
    ts_cmd->add_option("logfile", ts_logfile, "Path to JSON Lines log file")->required();
    ts_cmd->add_option("time_field", ts_time_field, "Time field name")->required();
    ts_cmd->add_option("-f,--filter", ts_filters, "Filter expressions");
    ts_cmd->add_option("-i,--interval", ts_interval, "Time interval (minute|hour|day|month)");

    // Top subcommand
    auto top_cmd = app.add_subcommand("top", "Show top N most frequent values for a field");
    std::string top_logfile, top_field;
    int top_n = 10;
    top_cmd->add_option("logfile", top_logfile, "Path to JSON Lines log file")->required();
    top_cmd->add_option("field", top_field, "Field to analyze")->required();
    top_cmd->add_option("-n,--top", top_n, "Number of top values");

    CLI11_PARSE(app, argc, argv);

    if (query_cmd->parsed()) {
        auto records = read_and_filter(q_logfile, q_filters);
        auto fields = q_fields_str.empty() ? std::vector<std::string>{} : split_csv(q_fields_str);
        int count = 0;
        for (const auto& rec : records) {
            if (q_limit > 0 && count >= q_limit) break;
            json output = fields.empty() ? rec : select_fields(rec, fields);
            std::cout << (q_pretty ? output.dump(2) : output.dump()) << "\n";
            count++;
        }
        std::cerr << "\n--- " << count << "/" << records.size() << " records matched ---\n";
    }
    else if (countby_cmd->parsed()) {
        auto records = read_and_filter(cb_logfile, cb_filters);
        std::map<std::string, int> counts;
        for (const auto& rec : records) {
            json val = get_nested_field(rec, cb_field);
            std::string key = val.is_null() ? "<null>" : json_to_string(val);
            counts[key]++;
        }
        std::vector<std::pair<std::string, int>> sorted_counts(counts.begin(), counts.end());
        std::sort(sorted_counts.begin(), sorted_counts.end(),
                  [](const auto& a, const auto& b) { return b.second < a.second; });
        for (const auto& [k, v] : sorted_counts) {
            std::cout << k << ": " << v << "\n";
        }
    }
    else if (stats_cmd->parsed()) {
        auto records = read_and_filter(st_logfile, st_filters);
        std::vector<double> values;
        for (const auto& rec : records) {
            json val = get_nested_field(rec, st_field);
            if (!val.is_null() && val.is_number()) values.push_back(val.get<double>());
        }
        if (values.empty()) {
            std::cout << "count: 0\nmin: 0\nmax: 0\navg: 0\nsum: 0\n";
        } else {
            double sum = std::accumulate(values.begin(), values.end(), 0.0);
            double min_v = *std::min_element(values.begin(), values.end());
            double max_v = *std::max_element(values.begin(), values.end());
            std::cout << "count: " << values.size() << "\n"
                      << "min: " << min_v << "\n"
                      << "max: " << max_v << "\n"
                      << "avg: " << (sum / values.size()) << "\n"
                      << "sum: " << sum << "\n";
        }
    }
    else if (ts_cmd->parsed()) {
        auto records = read_and_filter(ts_logfile, ts_filters);
        std::map<std::string, int> buckets;
        for (const auto& rec : records) {
            json val = get_nested_field(rec, ts_time_field);
            if (val.is_null()) continue;
            std::string bucket = format_time_bucket(json_to_string(val), ts_interval);
            if (!bucket.empty()) buckets[bucket]++;
        }
        for (const auto& [k, v] : buckets) {
            std::cout << k << ": " << v << "\n";
        }
    }
    else if (top_cmd->parsed()) {
        auto records = read_and_filter(top_logfile, {});
        std::map<std::string, int> counts;
        for (const auto& rec : records) {
            json val = get_nested_field(rec, top_field);
            if (!val.is_null()) counts[json_to_string(val)]++;
        }
        std::vector<std::pair<std::string, int>> sorted_counts(counts.begin(), counts.end());
        std::sort(sorted_counts.begin(), sorted_counts.end(),
                  [](const auto& a, const auto& b) { return b.second < a.second; });
        int shown = 0;
        for (const auto& [k, v] : sorted_counts) {
            if (shown >= top_n) break;
            std::cout << k << ": " << v << "\n";
            shown++;
        }
    }

    return 0;
}