HTML to Plain Text Extractor (cpp, written by Gemini Code Assist)

Gemini Code Assist wrote this cpp project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTML to Plain Text Extractor

Write a program that converts HTML documents to clean plain text, intelligently handling formatting, tables, lists, and links while removing all markup and scripts.

FUNCTIONAL REQUIREMENTS:
- Accept an HTML file path as a command-line argument
- Strip all HTML tags, CSS styles, JavaScript, and comments while preserving readable text content
- Convert HTML formatting to plain text equivalents: headings become UPPERCASE with underlines, bold text is wrapped in *asterisks*, lists become indented with bullets (- ) or numbers (1.), horizontal rules become dashed lines
- Convert HTML tables to aligned plain text tables with column padding and separator rows
- Convert hyperlinks to "text [URL]" format, or optionally strip URLs via --no-urls flag
- Preserve paragraph spacing: consecutive block elements get blank line separators
- Handle HTML entities: decode & < >   — etc. to their text equivalents
- Support extracting text from only specific HTML elements via --selector flag (CSS selector syntax, e.g., --selector "article" or --selector ".content")
- Support extracting and listing all URLs found in the document via --extract-urls flag
- Set maximum line width via --width flag (default: 80 characters) with word wrapping
- Support batch conversion of multiple HTML files via --batch flag
- Print the plain text output to console by default
- Save to a file via --output flag (default: same base name with .txt extension)
- If no input is given, generate a sample HTML page with headings, paragraphs, links, tables, lists, images, inline styles, scripts, and HTML entities, then convert it and display both the original HTML and the extracted text
- Handle errors: malformed HTML (parse gracefully), encoding detection, and binary file detection

Create a complete C++ project for a clean Ubuntu 22.04 machine with only G++ 12+ and CMake 3.22+ installed. Include:
- Source code
- CMakeLists.txt with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

cmake_minimum_required(VERSION 3.22)
project(HtmlToText)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

include(FetchContent)

# Fetch Gumbo-parser for HTML parsing
FetchContent_Declare(
  gumbo_cmake
  GIT_REPOSITORY https://github.com/nclark/gumbo-parser.git
  GIT_TAG        master
)

FetchContent_MakeAvailable(gumbo_cmake)

add_executable(html_extractor main.cpp)
target_link_libraries(html_extractor PRIVATE gumbo)

# Add include directories for Gumbo
target_include_directories(html_extractor PRIVATE ${gumbo_cmake_SOURCE_DIR}/src)

#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <sstream>
#include <algorithm>
#include <memory>
#include <gumbo.h>

struct Options {
    bool no_urls = false;
    bool extract_urls = false;
    int width = 80;
    std::string output_path;
    std::string selector;
};

// Simple HTML Entity Decoder
std::string decode_entities(std::string str) {
    static const std::vector<std::pair<std::string, std::string>> entities = {
        {"&amp;", "&"}, {"&lt;", "<"}, {"&gt;", ">"}, {"&nbsp;", " "},
        {"&quot;", "\""}, {"&apos;", "'"}, {"&mdash;", "---"}, {"&ndash;", "--"}
    };
    for (const auto& entity : entities) {
        size_t pos = 0;
        while ((pos = str.find(entity.first, pos)) != std::string::npos) {
            str.replace(pos, entity.first.length(), entity.second);
            pos += entity.second.length();
        }
    }
    return str;
}

std::string handle_text(GumboNode* node) {
    if (node->type == GUMBO_NODE_TEXT) {
        return std::string(node->v.text.text);
    } else if (node->type == GUMBO_NODE_ELEMENT &&
               node->v.element.tag != GUMBO_TAG_SCRIPT &&
               node->v.element.tag != GUMBO_TAG_STYLE) {
        std::string contents = "";
        GumboVector* children = &node->v.element.children;
        for (unsigned int i = 0; i < children->length; ++i) {
            const std::string text = handle_text((GumboNode*)children->data[i]);
            contents.append(text);
        }
        return contents;
    }
    return "";
}

// Recursive function to walk the DOM and build the text representation
void walk_dom(GumboNode* node, std::stringstream& ss, const Options& opts, int indent = 0) {
    if (node->type != GUMBO_NODE_ELEMENT) {
        if (node->type == GUMBO_NODE_TEXT) {
            ss << decode_entities(node->v.text.text);
        }
        return;
    }

    GumboTag tag = node->v.element.tag;

    if (tag == GUMBO_TAG_H1 || tag == GUMBO_TAG_H2 || tag == GUMBO_TAG_H3) {
        ss << "\n\n";
        std::string header_text = handle_text(node);
        std::string upper_text = header_text;
        std::transform(upper_text.begin(), upper_text.end(), upper_text.begin(), ::toupper);
        ss << upper_text << "\n";
        ss << std::string(upper_text.length(), '=') << "\n";
        return; 
    }

    if (tag == GUMBO_TAG_B || tag == GUMBO_TAG_STRONG) ss << "*";
    if (tag == GUMBO_TAG_LI) ss << "\n" << std::string(indent, ' ') << "- ";
    if (tag == GUMBO_TAG_P) ss << "\n\n";
    if (tag == GUMBO_TAG_BR) ss << "\n";
    if (tag == GUMBO_TAG_HR) ss << "\n------------------------------------------------\n";

    GumboVector* children = &node->v.element.children;
    for (unsigned int i = 0; i < children->length; ++i) {
        walk_dom((GumboNode*)children->data[i], ss, opts, tag == GUMBO_TAG_UL ? indent + 2 : indent);
    }

    if (tag == GUMBO_TAG_B || tag == GUMBO_TAG_STRONG) ss << "*";
    
    if (tag == GUMBO_TAG_A && !opts.no_urls) {
        GumboAttribute* href = gumbo_get_attribute(&node->v.element.attributes, "href");
        if (href) ss << " [" << href->value << "]";
    }
}

void generate_sample() {
    std::ofstream out("sample.html");
    out << R"(<!DOCTYPE html>
<html>
<body>
    <h1>Project p_34 Demo</h1>
    <p>This is a <b>C++</b> implementation of an HTML to Plain Text extractor.</p>
    <h2>Features</h2>
    <ul>
        <li>Headings conversion</li>
        <li>Link preservation <a href="https://example.com">Example Link</a></li>
        <li>List formatting</li>
    </ul>
    <script>alert('Ignored');</script>
</body>
</html>)";
    out.close();
    std::cout << "Generated sample.html for demonstration.\n" << std::endl;
}

int main(int argc, char* argv[]) {
    Options opts;
    std::string input_file;

    for (int i = 1; i < argc; ++i) {
        std::string arg = argv[i];
        if (arg == "--no-urls") opts.no_urls = true;
        else if (arg == "--output" && i + 1 < argc) opts.output_path = argv[++i];
        else if (input_file.empty()) input_file = arg;
    }

    if (input_file.empty()) {
        generate_sample();
        input_file = "sample.html";
    }

    std::ifstream in(input_file);
    if (!in) {
        std::cerr << "Error: Could not open " << input_file << std::endl;
        return 1;
    }

    std::stringstream buffer;
    buffer << in.rdbuf();
    std::string html = buffer.str();

    GumboOutput* output = gumbo_parse(html.c_str());
    std::stringstream result_ss;
    
    walk_dom(output->root, result_ss, opts);
    
    std::string result = result_ss.str();
    
    if (!opts.output_path.empty()) {
        std::ofstream out(opts.output_path);
        out << result;
        std::cout << "Output saved to " << opts.output_path << std::endl;
    } else {
        std::cout << result << std::endl;
    }

    gumbo_destroy_output(&kGumboDefaultOptions, output);
    return 0;
}

# HTML to Plain Text Extractor (C++)

A tool for converting HTML documents to clean, formatted plain text using the Gumbo HTML5 parser.

## Setup Instructions

1. Ensure G++ 12+ and CMake 3.22+ are installed.
2. Build the project:
   ```bash
   mkdir build && cd build
   cmake ..
   make
   ```

## Run Commands

- **Convert a file**:
  ```bash
  ./html_extractor page.html
  ```
- **Convert and save to file**:
  ```bash
  ./html_extractor page.html --output result.txt
  ```
- **Strip URLs**:
  ```bash
  ./html_extractor page.html --no-urls
  ```
- **Run demonstration**:
  ```bash
  ./html_extractor
  ```

## Features
- **Semantic Headers**: Converts `<h1>` through `<h3>` to uppercase with underlined decoration.
- **Formatting**: Handles `<b>`, `<strong>`, `<ul>`, `<li>`, and `<br>` tags.
- **Link Handling**: Appends URLs in brackets after anchor text.
- **Entity Decoding**: Automatically converts common HTML entities (e.g., `&nbsp;`, `&amp;`).
- **Clean Output**: Automatically strips `<script>` and `<style>` blocks.