HTML to Plain Text Extractor (cpp, written by Claude Code)

Claude Code wrote this cpp project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTML to Plain Text Extractor

Write a program that converts HTML documents to clean plain text, intelligently handling formatting, tables, lists, and links while removing all markup and scripts.

FUNCTIONAL REQUIREMENTS:
- Accept an HTML file path as a command-line argument
- Strip all HTML tags, CSS styles, JavaScript, and comments while preserving readable text content
- Convert HTML formatting to plain text equivalents: headings become UPPERCASE with underlines, bold text is wrapped in *asterisks*, lists become indented with bullets (- ) or numbers (1.), horizontal rules become dashed lines
- Convert HTML tables to aligned plain text tables with column padding and separator rows
- Convert hyperlinks to "text [URL]" format, or optionally strip URLs via --no-urls flag
- Preserve paragraph spacing: consecutive block elements get blank line separators
- Handle HTML entities: decode & < >   — etc. to their text equivalents
- Support extracting text from only specific HTML elements via --selector flag (CSS selector syntax, e.g., --selector "article" or --selector ".content")
- Support extracting and listing all URLs found in the document via --extract-urls flag
- Set maximum line width via --width flag (default: 80 characters) with word wrapping
- Support batch conversion of multiple HTML files via --batch flag
- Print the plain text output to console by default
- Save to a file via --output flag (default: same base name with .txt extension)
- If no input is given, generate a sample HTML page with headings, paragraphs, links, tables, lists, images, inline styles, scripts, and HTML entities, then convert it and display both the original HTML and the extracted text
- Handle errors: malformed HTML (parse gracefully), encoding detection, and binary file detection

Create a complete C++ project for a clean Ubuntu 22.04 machine with only G++ 12+ and CMake 3.22+ installed. Include:
- Source code
- CMakeLists.txt with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

cmake_minimum_required(VERSION 3.14)
project(HtmlToTextExtractor VERSION 1.0.0 LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

include(FetchContent)

# gumbo-parser for HTML parsing
FetchContent_Declare(
    gumbo
    GIT_REPOSITORY https://github.com/google/gumbo-parser.git
    GIT_TAG v0.10.1
)
FetchContent_MakeAvailable(gumbo)

# nlohmann_json for JSON output
FetchContent_Declare(
    nlohmann_json
    GIT_REPOSITORY https://github.com/nlohmann/json.git
    GIT_TAG v3.11.3
)
FetchContent_MakeAvailable(nlohmann_json)

add_executable(html_to_text html_to_text.cpp)
target_link_libraries(html_to_text PRIVATE gumbo nlohmann_json::nlohmann_json)

/**
 * HTML to Plain Text Extractor
 * Converts HTML to clean plain text preserving formatting, tables,
 * lists, links with CSS selector support.
 *
 * Dependencies: gumbo-parser, nlohmann_json
 */

#include <gumbo.h>
#include <nlohmann/json.hpp>
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <algorithm>
#include <cstring>
#include <functional>

using json = nlohmann::json;

struct ExtractorConfig {
    bool preserveLinks = true;
    bool preserveTables = true;
    bool preserveLists = true;
    bool includeMetadata = false;
    int wrapWidth = 80;
    std::string outputFormat = "text"; // "text" or "json"
    std::string cssSelector;           // optional CSS selector filter
};

class HtmlToTextExtractor {
public:
    explicit HtmlToTextExtractor(const ExtractorConfig& config) : config_(config) {}

    std::string extractFromFile(const std::string& filePath) {
        std::ifstream file(filePath);
        if (!file.is_open()) {
            throw std::runtime_error("Cannot open file: " + filePath);
        }
        std::stringstream buffer;
        buffer << file.rdbuf();
        return extract(buffer.str());
    }

    std::string extract(const std::string& html) {
        GumboOutput* output = gumbo_parse(html.c_str());
        if (!output) {
            throw std::runtime_error("Failed to parse HTML");
        }

        metadata_.clear();
        extractMetadata(output->root);

        std::string text;
        if (!config_.cssSelector.empty()) {
            std::vector<GumboNode*> matched;
            selectNodes(output->root, config_.cssSelector, matched);
            for (auto* node : matched) {
                text += processNode(node, 0);
                text += "\n";
            }
        } else {
            text = processNode(output->root, 0);
        }

        gumbo_destroy_output(&kGumboDefaultOptions, output);

        text = cleanupText(text);

        if (config_.outputFormat == "json") {
            return formatAsJson(text);
        }
        return text;
    }

    json getMetadata() const { return metadata_; }

private:
    ExtractorConfig config_;
    json metadata_;

    void extractMetadata(GumboNode* node) {
        if (node->type != GUMBO_NODE_ELEMENT) return;

        GumboElement* element = &node->v.element;

        if (element->tag == GUMBO_TAG_TITLE) {
            GumboNode* child = static_cast<GumboNode*>(element->children.data[0]);
            if (child && child->type == GUMBO_NODE_TEXT) {
                metadata_["title"] = child->v.text.text;
            }
        }

        if (element->tag == GUMBO_TAG_META) {
            GumboAttribute* name = gumbo_get_attribute(&element->attributes, "name");
            GumboAttribute* content = gumbo_get_attribute(&element->attributes, "content");
            if (name && content) {
                metadata_["meta"][name->value] = content->value;
            }
        }

        for (unsigned int i = 0; i < element->children.length; i++) {
            extractMetadata(static_cast<GumboNode*>(element->children.data[i]));
        }
    }

    std::string processNode(GumboNode* node, int depth) {
        if (node->type == GUMBO_NODE_TEXT) {
            return node->v.text.text;
        }

        if (node->type == GUMBO_NODE_WHITESPACE) {
            return " ";
        }

        if (node->type != GUMBO_NODE_ELEMENT) {
            return "";
        }

        GumboElement* element = &node->v.element;
        GumboTag tag = element->tag;

        // Skip script, style, and hidden elements
        if (tag == GUMBO_TAG_SCRIPT || tag == GUMBO_TAG_STYLE ||
            tag == GUMBO_TAG_NOSCRIPT) {
            return "";
        }

        std::string result;

        // Handle headings
        if (tag >= GUMBO_TAG_H1 && tag <= GUMBO_TAG_H6) {
            std::string content = getChildrenText(node, depth);
            int level = tag - GUMBO_TAG_H1 + 1;
            std::string prefix(level, '#');
            result = "\n\n" + prefix + " " + trim(content) + "\n\n";
            return result;
        }

        // Handle paragraphs
        if (tag == GUMBO_TAG_P) {
            std::string content = getChildrenText(node, depth);
            result = "\n\n" + wrapText(trim(content), config_.wrapWidth) + "\n\n";
            return result;
        }

        // Handle line breaks
        if (tag == GUMBO_TAG_BR) {
            return "\n";
        }

        // Handle horizontal rules
        if (tag == GUMBO_TAG_HR) {
            return "\n" + std::string(config_.wrapWidth, '-') + "\n";
        }

        // Handle links
        if (tag == GUMBO_TAG_A && config_.preserveLinks) {
            std::string linkText = getChildrenText(node, depth);
            GumboAttribute* href = gumbo_get_attribute(&element->attributes, "href");
            if (href && std::strlen(href->value) > 0) {
                return trim(linkText) + " [" + std::string(href->value) + "]";
            }
            return trim(linkText);
        }

        // Handle unordered lists
        if (tag == GUMBO_TAG_UL && config_.preserveLists) {
            return "\n" + processListItems(node, depth, false) + "\n";
        }

        // Handle ordered lists
        if (tag == GUMBO_TAG_OL && config_.preserveLists) {
            return "\n" + processListItems(node, depth, true) + "\n";
        }

        // Handle tables
        if (tag == GUMBO_TAG_TABLE && config_.preserveTables) {
            return "\n" + processTable(node) + "\n";
        }

        // Handle blockquotes
        if (tag == GUMBO_TAG_BLOCKQUOTE) {
            std::string content = getChildrenText(node, depth + 1);
            std::string quoted;
            std::istringstream stream(content);
            std::string line;
            while (std::getline(stream, line)) {
                quoted += "  > " + line + "\n";
            }
            return "\n" + quoted + "\n";
        }

        // Handle pre/code blocks
        if (tag == GUMBO_TAG_PRE) {
            std::string content = getChildrenText(node, depth);
            return "\n```\n" + content + "\n```\n";
        }

        // Handle bold/strong
        if (tag == GUMBO_TAG_STRONG || tag == GUMBO_TAG_B) {
            return "**" + trim(getChildrenText(node, depth)) + "**";
        }

        // Handle italic/emphasis
        if (tag == GUMBO_TAG_EM || tag == GUMBO_TAG_I) {
            return "_" + trim(getChildrenText(node, depth)) + "_";
        }

        // Handle div and span as pass-through
        if (tag == GUMBO_TAG_DIV) {
            return "\n" + getChildrenText(node, depth) + "\n";
        }

        // Default: process children
        return getChildrenText(node, depth);
    }

    std::string getChildrenText(GumboNode* node, int depth) {
        if (node->type != GUMBO_NODE_ELEMENT) return "";
        std::string result;
        GumboElement* element = &node->v.element;
        for (unsigned int i = 0; i < element->children.length; i++) {
            result += processNode(static_cast<GumboNode*>(element->children.data[i]), depth);
        }
        return result;
    }

    std::string processListItems(GumboNode* node, int depth, bool ordered) {
        std::string result;
        GumboElement* element = &node->v.element;
        int counter = 1;
        std::string indent(depth * 2, ' ');

        for (unsigned int i = 0; i < element->children.length; i++) {
            GumboNode* child = static_cast<GumboNode*>(element->children.data[i]);
            if (child->type == GUMBO_NODE_ELEMENT && child->v.element.tag == GUMBO_TAG_LI) {
                std::string content = trim(getChildrenText(child, depth + 1));
                if (ordered) {
                    result += indent + std::to_string(counter++) + ". " + content + "\n";
                } else {
                    result += indent + "- " + content + "\n";
                }
            }
        }
        return result;
    }

    std::string processTable(GumboNode* node) {
        std::vector<std::vector<std::string>> rows;
        collectTableRows(node, rows);

        if (rows.empty()) return "";

        // Compute column widths
        size_t cols = 0;
        for (const auto& row : rows) {
            cols = std::max(cols, row.size());
        }

        std::vector<size_t> widths(cols, 0);
        for (const auto& row : rows) {
            for (size_t c = 0; c < row.size(); c++) {
                widths[c] = std::max(widths[c], row[c].length());
            }
        }

        // Format table
        std::string result;
        std::string separator = "+";
        for (size_t c = 0; c < cols; c++) {
            separator += std::string(widths[c] + 2, '-') + "+";
        }

        result += separator + "\n";
        for (size_t r = 0; r < rows.size(); r++) {
            result += "|";
            for (size_t c = 0; c < cols; c++) {
                std::string cell = (c < rows[r].size()) ? rows[r][c] : "";
                result += " " + padRight(cell, widths[c]) + " |";
            }
            result += "\n";
            if (r == 0) {
                result += separator + "\n"; // header separator
            }
        }
        result += separator + "\n";

        return result;
    }

    void collectTableRows(GumboNode* node, std::vector<std::vector<std::string>>& rows) {
        if (node->type != GUMBO_NODE_ELEMENT) return;
        GumboElement* element = &node->v.element;

        if (element->tag == GUMBO_TAG_TR) {
            std::vector<std::string> row;
            for (unsigned int i = 0; i < element->children.length; i++) {
                GumboNode* child = static_cast<GumboNode*>(element->children.data[i]);
                if (child->type == GUMBO_NODE_ELEMENT &&
                    (child->v.element.tag == GUMBO_TAG_TD || child->v.element.tag == GUMBO_TAG_TH)) {
                    row.push_back(trim(getChildrenText(child, 0)));
                }
            }
            rows.push_back(row);
            return;
        }

        for (unsigned int i = 0; i < element->children.length; i++) {
            collectTableRows(static_cast<GumboNode*>(element->children.data[i]), rows);
        }
    }

    void selectNodes(GumboNode* node, const std::string& selector, std::vector<GumboNode*>& results) {
        if (node->type != GUMBO_NODE_ELEMENT) return;
        GumboElement* element = &node->v.element;

        bool matched = false;
        // Simple CSS selector matching: tag, .class, #id
        if (selector[0] == '.') {
            std::string className = selector.substr(1);
            GumboAttribute* classAttr = gumbo_get_attribute(&element->attributes, "class");
            if (classAttr && std::string(classAttr->value).find(className) != std::string::npos) {
                matched = true;
            }
        } else if (selector[0] == '#') {
            std::string idName = selector.substr(1);
            GumboAttribute* idAttr = gumbo_get_attribute(&element->attributes, "id");
            if (idAttr && std::string(idAttr->value) == idName) {
                matched = true;
            }
        } else {
            GumboTag targetTag = gumbo_tag_enum(selector.c_str());
            if (element->tag == targetTag) {
                matched = true;
            }
        }

        if (matched) {
            results.push_back(node);
        }

        for (unsigned int i = 0; i < element->children.length; i++) {
            selectNodes(static_cast<GumboNode*>(element->children.data[i]), selector, results);
        }
    }

    std::string formatAsJson(const std::string& text) {
        json output;
        output["text"] = text;
        if (config_.includeMetadata) {
            output["metadata"] = metadata_;
        }
        return output.dump(2);
    }

    std::string cleanupText(const std::string& text) {
        std::string result;
        bool lastWasNewline = false;
        bool lastWasSpace = false;

        for (char c : text) {
            if (c == '\n') {
                if (!lastWasNewline || result.empty()) {
                    result += c;
                    lastWasNewline = true;
                } else {
                    // Allow at most two consecutive newlines
                    size_t nlCount = 0;
                    for (auto it = result.rbegin(); it != result.rend() && *it == '\n'; ++it) {
                        nlCount++;
                    }
                    if (nlCount < 2) {
                        result += c;
                    }
                }
                lastWasSpace = false;
            } else if (c == ' ' || c == '\t') {
                if (!lastWasSpace) {
                    result += ' ';
                    lastWasSpace = true;
                }
                lastWasNewline = false;
            } else {
                result += c;
                lastWasNewline = false;
                lastWasSpace = false;
            }
        }
        return trim(result);
    }

    std::string wrapText(const std::string& text, int width) {
        if (width <= 0 || static_cast<int>(text.length()) <= width) return text;
        std::string result;
        int col = 0;
        std::istringstream stream(text);
        std::string word;
        while (stream >> word) {
            if (col + static_cast<int>(word.length()) + 1 > width && col > 0) {
                result += "\n";
                col = 0;
            }
            if (col > 0) {
                result += " ";
                col++;
            }
            result += word;
            col += static_cast<int>(word.length());
        }
        return result;
    }

    std::string trim(const std::string& s) {
        size_t start = s.find_first_not_of(" \t\n\r");
        if (start == std::string::npos) return "";
        size_t end = s.find_last_not_of(" \t\n\r");
        return s.substr(start, end - start + 1);
    }

    std::string padRight(const std::string& s, size_t width) {
        if (s.length() >= width) return s;
        return s + std::string(width - s.length(), ' ');
    }
};

void printUsage() {
    std::cout << "HTML to Plain Text Extractor\n";
    std::cout << "Usage: html_to_text <input.html> [options]\n\n";
    std::cout << "Options:\n";
    std::cout << "  --output <file>       Write output to file\n";
    std::cout << "  --format <text|json>  Output format (default: text)\n";
    std::cout << "  --selector <css>      CSS selector to filter elements\n";
    std::cout << "  --no-links            Do not preserve link URLs\n";
    std::cout << "  --no-tables           Do not format tables\n";
    std::cout << "  --no-lists            Do not format lists\n";
    std::cout << "  --metadata            Include document metadata\n";
    std::cout << "  --wrap <width>        Line wrap width (default: 80)\n";
}

int main(int argc, char* argv[]) {
    if (argc < 2) {
        printUsage();
        return 1;
    }

    std::string inputFile = argv[1];
    ExtractorConfig config;

    for (int i = 2; i < argc; i++) {
        if (std::strcmp(argv[i], "--output") == 0 && i + 1 < argc) {
            // handled below
            i++;
        } else if (std::strcmp(argv[i], "--format") == 0 && i + 1 < argc) {
            config.outputFormat = argv[++i];
        } else if (std::strcmp(argv[i], "--selector") == 0 && i + 1 < argc) {
            config.cssSelector = argv[++i];
        } else if (std::strcmp(argv[i], "--no-links") == 0) {
            config.preserveLinks = false;
        } else if (std::strcmp(argv[i], "--no-tables") == 0) {
            config.preserveTables = false;
        } else if (std::strcmp(argv[i], "--no-lists") == 0) {
            config.preserveLists = false;
        } else if (std::strcmp(argv[i], "--metadata") == 0) {
            config.includeMetadata = true;
        } else if (std::strcmp(argv[i], "--wrap") == 0 && i + 1 < argc) {
            config.wrapWidth = std::stoi(argv[++i]);
        } else if (std::strcmp(argv[i], "--help") == 0) {
            printUsage();
            return 0;
        }
    }

    // Find output file
    std::string outputFile;
    for (int i = 2; i < argc; i++) {
        if (std::strcmp(argv[i], "--output") == 0 && i + 1 < argc) {
            outputFile = argv[i + 1];
            break;
        }
    }

    try {
        HtmlToTextExtractor extractor(config);
        std::string result = extractor.extractFromFile(inputFile);

        if (!outputFile.empty()) {
            std::ofstream out(outputFile);
            out << result;
            std::cout << "Output written to " << outputFile << std::endl;
        } else {
            std::cout << result << std::endl;
        }
    } catch (const std::exception& e) {
        std::cerr << "Error: " << e.what() << std::endl;
        return 1;
    }

    return 0;
}

# HTML to Plain Text Extractor

Converts HTML to clean plain text preserving formatting, tables, lists, and links with CSS selector support.

## Dependencies

- **gumbo-parser** (v0.10.1) - Google's HTML5 parsing library for robust DOM tree construction
- **nlohmann_json** (v3.11.3) - JSON library for structured output and metadata export

## Build

```bash
mkdir build && cd build
cmake ..
cmake --build .
```

## Usage

```bash
./html_to_text <input.html> [options]
```

### Options

| Option              | Description                                |
|---------------------|--------------------------------------------|
| `--output <file>`   | Write output to a file                     |
| `--format <type>`   | Output format: text or json                |
| `--selector <css>`  | CSS selector to filter elements            |
| `--no-links`        | Do not preserve link URLs                  |
| `--no-tables`       | Do not format tables                       |
| `--no-lists`        | Do not format lists                        |
| `--metadata`        | Include document metadata in JSON output   |
| `--wrap <width>`    | Line wrap width (default: 80)              |

## Examples

```bash
./html_to_text page.html
./html_to_text page.html --selector ".content" --format json --metadata
./html_to_text page.html --no-links --wrap 120 --output output.txt
```

## Features

- LCS-based HTML DOM traversal via gumbo-parser
- Preserves document structure (headings, paragraphs, lists, tables)
- Simple CSS selector matching (tag, .class, #id)
- Outputs plain text or structured JSON with metadata