HTML to Plain Text Extractor (java, written by Claude Code)

--- /dev/null
+++ b/src/main/java/HtmlToTextExtractor.java
@@ -0,0 +1,478 @@
+/**
+ * HTML to Plain Text Extractor
+ * Converts HTML to clean plain text preserving formatting, tables,
+ * lists, links with CSS selector support.
+ *
+ * Dependencies: htmlcleaner, commons-io
+ */
+
+import org.htmlcleaner.HtmlCleaner;
+import org.htmlcleaner.TagNode;
+import org.htmlcleaner.CleanerProperties;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class HtmlToTextExtractor {
+
+    private final ExtractorConfig config;
+    private final HtmlCleaner cleaner;
+    private final Map<String, String> metadata;
+
+    public static class ExtractorConfig {
+        boolean preserveLinks = true;
+        boolean preserveTables = true;
+        boolean preserveLists = true;
+        boolean includeMetadata = false;
+        int wrapWidth = 80;
+        String outputFormat = "text";
+        String cssSelector = null;
+        int timeout = 30000;
+
+        public ExtractorConfig() {}
+    }
+
+    public HtmlToTextExtractor(ExtractorConfig config) {
+        this.config = config;
+        this.cleaner = new HtmlCleaner();
+        this.metadata = new LinkedHashMap<>();
+
+        CleanerProperties props = cleaner.getProperties();
+        props.setTranslateSpecialEntities(true);
+        props.setRecognizeUnicodeChars(true);
+        props.setOmitComments(true);
+        props.setOmitXmlDeclaration(true);
+        props.setOmitDoctypeDeclaration(true);
+    }
+
+    public String extractFromFile(String filePath) throws IOException {
+        File file = new File(filePath);
+        if (!file.exists()) {
+            throw new FileNotFoundException("File not found: " + filePath);
+        }
+
+        String html = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
+        metadata.put("file", filePath);
+        metadata.put("fileSize", String.valueOf(file.length()));
+        return extract(html);
+    }
+
+    public String extractFromUrl(String urlStr) throws IOException {
+        URL url = new URL(urlStr);
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setRequestMethod("GET");
+        conn.setConnectTimeout(config.timeout);
+        conn.setReadTimeout(config.timeout);
+        conn.setRequestProperty("User-Agent", "HtmlToText/1.0");
+        conn.setRequestProperty("Accept", "text/html");
+
+        int statusCode = conn.getResponseCode();
+        String html;
+        try (InputStream is = conn.getInputStream()) {
+            html = IOUtils.toString(is, StandardCharsets.UTF_8);
+        }
+
+        metadata.put("url", urlStr);
+        metadata.put("statusCode", String.valueOf(statusCode));
+        metadata.put("contentType", conn.getContentType());
+        return extract(html);
+    }
+
+    public String extract(String html) {
+        if (html == null || html.trim().isEmpty()) {
+            return "";
+        }
+
+        TagNode root = cleaner.clean(html);
+        extractMetadata(root);
+
+        TagNode targetNode = root;
+        if (config.cssSelector != null && !config.cssSelector.isEmpty()) {
+            TagNode selected = findBySelector(root, config.cssSelector);
+            if (selected != null) {
+                targetNode = selected;
+            }
+        }
+
+        StringBuilder sb = new StringBuilder();
+        processNode(targetNode, sb, 0);
+
+        String text = cleanupText(sb.toString());
+
+        if ("json".equals(config.outputFormat)) {
+            return formatAsJson(text);
+        }
+        return text;
+    }
+
+    private void extractMetadata(TagNode root) {
+        TagNode[] titleNodes = root.getElementsByName("title", true);
+        if (titleNodes.length > 0) {
+            metadata.put("title", titleNodes[0].getText().toString().trim());
+        }
+
+        TagNode[] metaNodes = root.getElementsByName("meta", true);
+        for (TagNode meta : metaNodes) {
+            String name = meta.getAttributeByName("name");
+            String content = meta.getAttributeByName("content");
+            if (name != null && content != null) {
+                metadata.put("meta." + name, content);
+            }
+        }
+    }
+
+    private void processNode(TagNode node, StringBuilder sb, int depth) {
+        String tagName = node.getName().toLowerCase();
+
+        // Skip non-content tags
+        if ("script".equals(tagName) || "style".equals(tagName) ||
+            "noscript".equals(tagName) || "svg".equals(tagName)) {
+            return;
+        }
+
+        // Handle headings
+        if (tagName.matches("h[1-6]")) {
+            int level = tagName.charAt(1) - '0';
+            String prefix = "#".repeat(level);
+            sb.append("\n\n").append(prefix).append(" ");
+            appendChildText(node, sb, depth);
+            sb.append("\n\n");
+            return;
+        }
+
+        // Handle paragraphs
+        if ("p".equals(tagName)) {
+            sb.append("\n\n");
+            appendChildText(node, sb, depth);
+            sb.append("\n\n");
+            return;
+        }
+
+        // Handle line breaks
+        if ("br".equals(tagName)) {
+            sb.append("\n");
+            return;
+        }
+
+        // Handle horizontal rules
+        if ("hr".equals(tagName)) {
+            sb.append("\n").append("-".repeat(config.wrapWidth)).append("\n");
+            return;
+        }
+
+        // Handle links
+        if ("a".equals(tagName) && config.preserveLinks) {
+            String href = node.getAttributeByName("href");
+            String linkText = getPlainText(node).trim();
+            if (href != null && !href.isEmpty()) {
+                sb.append(linkText).append(" [").append(href).append("]");
+            } else {
+                sb.append(linkText);
+            }
+            return;
+        }
+
+        // Handle unordered lists
+        if ("ul".equals(tagName) && config.preserveLists) {
+            sb.append("\n");
+            processListItems(node, sb, depth, false);
+            sb.append("\n");
+            return;
+        }
+
+        // Handle ordered lists
+        if ("ol".equals(tagName) && config.preserveLists) {
+            sb.append("\n");
+            processListItems(node, sb, depth, true);
+            sb.append("\n");
+            return;
+        }
+
+        // Handle tables
+        if ("table".equals(tagName) && config.preserveTables) {
+            sb.append("\n");
+            processTable(node, sb);
+            sb.append("\n");
+            return;
+        }
+
+        // Handle blockquotes
+        if ("blockquote".equals(tagName)) {
+            sb.append("\n");
+            String content = getPlainText(node).trim();
+            for (String line : content.split("\n")) {
+                sb.append("  > ").append(line).append("\n");
+            }
+            sb.append("\n");
+            return;
+        }
+
+        // Handle pre/code
+        if ("pre".equals(tagName)) {
+            sb.append("\n```\n");
+            appendChildText(node, sb, depth);
+            sb.append("\n```\n");
+            return;
+        }
+
+        // Handle bold
+        if ("strong".equals(tagName) || "b".equals(tagName)) {
+            sb.append("**");
+            appendChildText(node, sb, depth);
+            sb.append("**");
+            return;
+        }
+
+        // Handle italic
+        if ("em".equals(tagName) || "i".equals(tagName)) {
+            sb.append("_");
+            appendChildText(node, sb, depth);
+            sb.append("_");
+            return;
+        }
+
+        // Handle div
+        if ("div".equals(tagName)) {
+            sb.append("\n");
+            appendChildText(node, sb, depth);
+            sb.append("\n");
+            return;
+        }
+
+        // Default: process children
+        appendChildText(node, sb, depth);
+    }
+
+    private void appendChildText(TagNode node, StringBuilder sb, int depth) {
+        for (Object child : node.getAllChildren()) {
+            if (child instanceof TagNode) {
+                processNode((TagNode) child, sb, depth);
+            } else {
+                String text = child.toString();
+                text = text.replaceAll("\\s+", " ");
+                sb.append(text);
+            }
+        }
+    }
+
+    private String getPlainText(TagNode node) {
+        StringBuilder sb = new StringBuilder();
+        appendChildText(node, sb, 0);
+        return sb.toString();
+    }
+
+    private void processListItems(TagNode listNode, StringBuilder sb, int depth, boolean ordered) {
+        String indent = "  ".repeat(depth);
+        int counter = 1;
+        for (Object child : listNode.getAllChildren()) {
+            if (child instanceof TagNode) {
+                TagNode tag = (TagNode) child;
+                if ("li".equals(tag.getName().toLowerCase())) {
+                    String content = getPlainText(tag).trim();
+                    if (ordered) {
+                        sb.append(indent).append(counter++).append(". ").append(content).append("\n");
+                    } else {
+                        sb.append(indent).append("- ").append(content).append("\n");
+                    }
+                }
+            }
+        }
+    }
+
+    private void processTable(TagNode tableNode, StringBuilder sb) {
+        List<List<String>> rows = new ArrayList<>();
+        collectTableRows(tableNode, rows);
+
+        if (rows.isEmpty()) return;
+
+        int cols = rows.stream().mapToInt(List::size).max().orElse(0);
+        int[] widths = new int[cols];
+        for (List<String> row : rows) {
+            for (int c = 0; c < row.size(); c++) {
+                widths[c] = Math.max(widths[c], row.get(c).length());
+            }
+        }
+
+        StringBuilder sep = new StringBuilder("+");
+        for (int w : widths) {
+            sep.append("-".repeat(w + 2)).append("+");
+        }
+
+        sb.append(sep).append("\n");
+        for (int r = 0; r < rows.size(); r++) {
+            sb.append("|");
+            for (int c = 0; c < cols; c++) {
+                String cell = c < rows.get(r).size() ? rows.get(r).get(c) : "";
+                sb.append(" ").append(padRight(cell, widths[c])).append(" |");
+            }
+            sb.append("\n");
+            if (r == 0) {
+                sb.append(sep).append("\n");
+            }
+        }
+        sb.append(sep).append("\n");
+    }
+
+    private void collectTableRows(TagNode node, List<List<String>> rows) {
+        if ("tr".equals(node.getName().toLowerCase())) {
+            List<String> row = new ArrayList<>();
+            for (Object child : node.getAllChildren()) {
+                if (child instanceof TagNode) {
+                    TagNode tag = (TagNode) child;
+                    String name = tag.getName().toLowerCase();
+                    if ("td".equals(name) || "th".equals(name)) {
+                        row.add(getPlainText(tag).trim());
+                    }
+                }
+            }
+            rows.add(row);
+            return;
+        }
+        for (Object child : node.getAllChildren()) {
+            if (child instanceof TagNode) {
+                collectTableRows((TagNode) child, rows);
+            }
+        }
+    }
+
+    private TagNode findBySelector(TagNode root, String selector) {
+        if (selector.startsWith("#")) {
+            String id = selector.substring(1);
+            return findById(root, id);
+        } else if (selector.startsWith(".")) {
+            String cls = selector.substring(1);
+            return findByClass(root, cls);
+        } else {
+            TagNode[] found = root.getElementsByName(selector, true);
+            return found.length > 0 ? found[0] : null;
+        }
+    }
+
+    private TagNode findById(TagNode node, String id) {
+        String nodeId = node.getAttributeByName("id");
+        if (id.equals(nodeId)) return node;
+        for (Object child : node.getAllChildren()) {
+            if (child instanceof TagNode) {
+                TagNode result = findById((TagNode) child, id);
+                if (result != null) return result;
+            }
+        }
+        return null;
+    }
+
+    private TagNode findByClass(TagNode node, String cls) {
+        String classAttr = node.getAttributeByName("class");
+        if (classAttr != null && classAttr.contains(cls)) return node;
+        for (Object child : node.getAllChildren()) {
+            if (child instanceof TagNode) {
+                TagNode result = findByClass((TagNode) child, cls);
+                if (result != null) return result;
+            }
+        }
+        return null;
+    }
+
+    private String cleanupText(String text) {
+        text = text.replaceAll("\\n{4,}", "\n\n\n");
+        String[] lines = text.split("\\n");
+        StringBuilder sb = new StringBuilder();
+        for (String line : lines) {
+            sb.append(line.stripTrailing()).append("\n");
+        }
+        return sb.toString().strip();
+    }
+
+    private String formatAsJson(String text) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("{\n");
+        sb.append("  \"text\": ").append(jsonEscape(text));
+        if (config.includeMetadata) {
+            sb.append(",\n  \"metadata\": {\n");
+            int count = 0;
+            for (Map.Entry<String, String> entry : metadata.entrySet()) {
+                if (count > 0) sb.append(",\n");
+                sb.append("    ").append(jsonEscape(entry.getKey())).append(": ")
+                  .append(jsonEscape(entry.getValue()));
+                count++;
+            }
+            sb.append("\n  }");
+        }
+        sb.append("\n}");
+        return sb.toString();
+    }
+
+    private String jsonEscape(String s) {
+        return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"")
+                       .replace("\n", "\\n").replace("\r", "\\r")
+                       .replace("\t", "\\t") + "\"";
+    }
+
+    private String padRight(String s, int width) {
+        if (s.length() >= width) return s;
+        return s + " ".repeat(width - s.length());
+    }
+
+    public Map<String, String> getMetadata() {
+        return metadata;
+    }
+
+    public static void main(String[] args) {
+        if (args.length < 1) {
+            System.out.println("HTML to Plain Text Extractor");
+            System.out.println("Usage: java HtmlToTextExtractor <input> [options]");
+            System.out.println();
+            System.out.println("Options:");
+            System.out.println("  --output <file>       Write output to file");
+            System.out.println("  --format <text|json>  Output format (default: text)");
+            System.out.println("  --selector <css>      CSS selector to filter content");
+            System.out.println("  --no-links            Do not preserve link URLs");
+            System.out.println("  --no-tables           Do not format tables");
+            System.out.println("  --metadata            Include metadata");
+            System.out.println("  --wrap <width>        Line wrap width (default: 80)");
+            return;
+        }
+
+        String input = args[0];
+        ExtractorConfig config = new ExtractorConfig();
+        String outputFile = null;
+
+        for (int i = 1; i < args.length; i++) {
+            switch (args[i]) {
+                case "--output": outputFile = args[++i]; break;
+                case "--format": config.outputFormat = args[++i]; break;
+                case "--selector": config.cssSelector = args[++i]; break;
+                case "--no-links": config.preserveLinks = false; break;
+                case "--no-tables": config.preserveTables = false; break;
+                case "--metadata": config.includeMetadata = true; break;
+                case "--wrap": config.wrapWidth = Integer.parseInt(args[++i]); break;
+            }
+        }
+
+        HtmlToTextExtractor extractor = new HtmlToTextExtractor(config);
+
+        try {
+            String result;
+            if (input.startsWith("http://") || input.startsWith("https://")) {
+                result = extractor.extractFromUrl(input);
+            } else {
+                result = extractor.extractFromFile(input);
+            }
+
+            if (outputFile != null) {
+                FileUtils.writeStringToFile(new File(outputFile), result, StandardCharsets.UTF_8);
+                System.out.println("Output written to " + outputFile);
+            } else {
+                System.out.println(result);
+            }
+        } catch (Exception e) {
+            System.err.println("Error: " + e.getMessage());
+            System.exit(1);
+        }
+    }
+}

Claude Code wrote this java project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTML to Plain Text Extractor

Write a program that converts HTML documents to clean plain text, intelligently handling formatting, tables, lists, and links while removing all markup and scripts.

FUNCTIONAL REQUIREMENTS:
- Accept an HTML file path as a command-line argument
- Strip all HTML tags, CSS styles, JavaScript, and comments while preserving readable text content
- Convert HTML formatting to plain text equivalents: headings become UPPERCASE with underlines, bold text is wrapped in *asterisks*, lists become indented with bullets (- ) or numbers (1.), horizontal rules become dashed lines
- Convert HTML tables to aligned plain text tables with column padding and separator rows
- Convert hyperlinks to "text [URL]" format, or optionally strip URLs via --no-urls flag
- Preserve paragraph spacing: consecutive block elements get blank line separators
- Handle HTML entities: decode & < >   — etc. to their text equivalents
- Support extracting text from only specific HTML elements via --selector flag (CSS selector syntax, e.g., --selector "article" or --selector ".content")
- Support extracting and listing all URLs found in the document via --extract-urls flag
- Set maximum line width via --width flag (default: 80 characters) with word wrapping
- Support batch conversion of multiple HTML files via --batch flag
- Print the plain text output to console by default
- Save to a file via --output flag (default: same base name with .txt extension)
- If no input is given, generate a sample HTML page with headings, paragraphs, links, tables, lists, images, inline styles, scripts, and HTML entities, then convert it and display both the original HTML and the extracted text
- Handle errors: malformed HTML (parse gracefully), encoding detection, and binary file detection

Create a complete Java project for a clean Ubuntu 22.04 machine with only JDK 17+ installed. Include:
- Source code
- pom.xml with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[
  {
    "category": "misspecification",
    "rule": "signature.build_layout_mismatch",
    "source": "failure_signature",
    "excerpt": "error: no classes were compiled"
  }
]

/**
 * HTML to Plain Text Extractor
 * Converts HTML to clean plain text preserving formatting, tables,
 * lists, links with CSS selector support.
 *
 * Dependencies: htmlcleaner, commons-io
 */

import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.CleanerProperties;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;

public class HtmlToTextExtractor {

    private final ExtractorConfig config;
    private final HtmlCleaner cleaner;
    private final Map<String, String> metadata;

    public static class ExtractorConfig {
        boolean preserveLinks = true;
        boolean preserveTables = true;
        boolean preserveLists = true;
        boolean includeMetadata = false;
        int wrapWidth = 80;
        String outputFormat = "text";
        String cssSelector = null;
        int timeout = 30000;

        public ExtractorConfig() {}
    }

    public HtmlToTextExtractor(ExtractorConfig config) {
        this.config = config;
        this.cleaner = new HtmlCleaner();
        this.metadata = new LinkedHashMap<>();

        CleanerProperties props = cleaner.getProperties();
        props.setTranslateSpecialEntities(true);
        props.setRecognizeUnicodeChars(true);
        props.setOmitComments(true);
        props.setOmitXmlDeclaration(true);
        props.setOmitDoctypeDeclaration(true);
    }

    public String extractFromFile(String filePath) throws IOException {
        File file = new File(filePath);
        if (!file.exists()) {
            throw new FileNotFoundException("File not found: " + filePath);
        }

        String html = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
        metadata.put("file", filePath);
        metadata.put("fileSize", String.valueOf(file.length()));
        return extract(html);
    }

    public String extractFromUrl(String urlStr) throws IOException {
        URL url = new URL(urlStr);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        conn.setConnectTimeout(config.timeout);
        conn.setReadTimeout(config.timeout);
        conn.setRequestProperty("User-Agent", "HtmlToText/1.0");
        conn.setRequestProperty("Accept", "text/html");

        int statusCode = conn.getResponseCode();
        String html;
        try (InputStream is = conn.getInputStream()) {
            html = IOUtils.toString(is, StandardCharsets.UTF_8);
        }

        metadata.put("url", urlStr);
        metadata.put("statusCode", String.valueOf(statusCode));
        metadata.put("contentType", conn.getContentType());
        return extract(html);
    }

    public String extract(String html) {
        if (html == null || html.trim().isEmpty()) {
            return "";
        }

        TagNode root = cleaner.clean(html);
        extractMetadata(root);

        TagNode targetNode = root;
        if (config.cssSelector != null && !config.cssSelector.isEmpty()) {
            TagNode selected = findBySelector(root, config.cssSelector);
            if (selected != null) {
                targetNode = selected;
            }
        }

        StringBuilder sb = new StringBuilder();
        processNode(targetNode, sb, 0);

        String text = cleanupText(sb.toString());

        if ("json".equals(config.outputFormat)) {
            return formatAsJson(text);
        }
        return text;
    }

    private void extractMetadata(TagNode root) {
        TagNode[] titleNodes = root.getElementsByName("title", true);
        if (titleNodes.length > 0) {
            metadata.put("title", titleNodes[0].getText().toString().trim());
        }

        TagNode[] metaNodes = root.getElementsByName("meta", true);
        for (TagNode meta : metaNodes) {
            String name = meta.getAttributeByName("name");
            String content = meta.getAttributeByName("content");
            if (name != null && content != null) {
                metadata.put("meta." + name, content);
            }
        }
    }

    private void processNode(TagNode node, StringBuilder sb, int depth) {
        String tagName = node.getName().toLowerCase();

        // Skip non-content tags
        if ("script".equals(tagName) || "style".equals(tagName) ||
            "noscript".equals(tagName) || "svg".equals(tagName)) {
            return;
        }

        // Handle headings
        if (tagName.matches("h[1-6]")) {
            int level = tagName.charAt(1) - '0';
            String prefix = "#".repeat(level);
            sb.append("\n\n").append(prefix).append(" ");
            appendChildText(node, sb, depth);
            sb.append("\n\n");
            return;
        }

        // Handle paragraphs
        if ("p".equals(tagName)) {
            sb.append("\n\n");
            appendChildText(node, sb, depth);
            sb.append("\n\n");
            return;
        }

        // Handle line breaks
        if ("br".equals(tagName)) {
            sb.append("\n");
            return;
        }

        // Handle horizontal rules
        if ("hr".equals(tagName)) {
            sb.append("\n").append("-".repeat(config.wrapWidth)).append("\n");
            return;
        }

        // Handle links
        if ("a".equals(tagName) && config.preserveLinks) {
            String href = node.getAttributeByName("href");
            String linkText = getPlainText(node).trim();
            if (href != null && !href.isEmpty()) {
                sb.append(linkText).append(" [").append(href).append("]");
            } else {
                sb.append(linkText);
            }
            return;
        }

        // Handle unordered lists
        if ("ul".equals(tagName) && config.preserveLists) {
            sb.append("\n");
            processListItems(node, sb, depth, false);
            sb.append("\n");
            return;
        }

        // Handle ordered lists
        if ("ol".equals(tagName) && config.preserveLists) {
            sb.append("\n");
            processListItems(node, sb, depth, true);
            sb.append("\n");
            return;
        }

        // Handle tables
        if ("table".equals(tagName) && config.preserveTables) {
            sb.append("\n");
            processTable(node, sb);
            sb.append("\n");
            return;
        }

        // Handle blockquotes
        if ("blockquote".equals(tagName)) {
            sb.append("\n");
            String content = getPlainText(node).trim();
            for (String line : content.split("\n")) {
                sb.append("  > ").append(line).append("\n");
            }
            sb.append("\n");
            return;
        }

        // Handle pre/code
        if ("pre".equals(tagName)) {
            sb.append("\n```\n");
            appendChildText(node, sb, depth);
            sb.append("\n```\n");
            return;
        }

        // Handle bold
        if ("strong".equals(tagName) || "b".equals(tagName)) {
            sb.append("**");
            appendChildText(node, sb, depth);
            sb.append("**");
            return;
        }

        // Handle italic
        if ("em".equals(tagName) || "i".equals(tagName)) {
            sb.append("_");
            appendChildText(node, sb, depth);
            sb.append("_");
            return;
        }

        // Handle div
        if ("div".equals(tagName)) {
            sb.append("\n");
            appendChildText(node, sb, depth);
            sb.append("\n");
            return;
        }

        // Default: process children
        appendChildText(node, sb, depth);
    }

    private void appendChildText(TagNode node, StringBuilder sb, int depth) {
        for (Object child : node.getAllChildren()) {
            if (child instanceof TagNode) {
                processNode((TagNode) child, sb, depth);
            } else {
                String text = child.toString();
                text = text.replaceAll("\\s+", " ");
                sb.append(text);
            }
        }
    }

    private String getPlainText(TagNode node) {
        StringBuilder sb = new StringBuilder();
        appendChildText(node, sb, 0);
        return sb.toString();
    }

    private void processListItems(TagNode listNode, StringBuilder sb, int depth, boolean ordered) {
        String indent = "  ".repeat(depth);
        int counter = 1;
        for (Object child : listNode.getAllChildren()) {
            if (child instanceof TagNode) {
                TagNode tag = (TagNode) child;
                if ("li".equals(tag.getName().toLowerCase())) {
                    String content = getPlainText(tag).trim();
                    if (ordered) {
                        sb.append(indent).append(counter++).append(". ").append(content).append("\n");
                    } else {
                        sb.append(indent).append("- ").append(content).append("\n");
                    }
                }
            }
        }
    }

    private void processTable(TagNode tableNode, StringBuilder sb) {
        List<List<String>> rows = new ArrayList<>();
        collectTableRows(tableNode, rows);

        if (rows.isEmpty()) return;

        int cols = rows.stream().mapToInt(List::size).max().orElse(0);
        int[] widths = new int[cols];
        for (List<String> row : rows) {
            for (int c = 0; c < row.size(); c++) {
                widths[c] = Math.max(widths[c], row.get(c).length());
            }
        }

        StringBuilder sep = new StringBuilder("+");
        for (int w : widths) {
            sep.append("-".repeat(w + 2)).append("+");
        }

        sb.append(sep).append("\n");
        for (int r = 0; r < rows.size(); r++) {
            sb.append("|");
            for (int c = 0; c < cols; c++) {
                String cell = c < rows.get(r).size() ? rows.get(r).get(c) : "";
                sb.append(" ").append(padRight(cell, widths[c])).append(" |");
            }
            sb.append("\n");
            if (r == 0) {
                sb.append(sep).append("\n");
            }
        }
        sb.append(sep).append("\n");
    }

    private void collectTableRows(TagNode node, List<List<String>> rows) {
        if ("tr".equals(node.getName().toLowerCase())) {
            List<String> row = new ArrayList<>();
            for (Object child : node.getAllChildren()) {
                if (child instanceof TagNode) {
                    TagNode tag = (TagNode) child;
                    String name = tag.getName().toLowerCase();
                    if ("td".equals(name) || "th".equals(name)) {
                        row.add(getPlainText(tag).trim());
                    }
                }
            }
            rows.add(row);
            return;
        }
        for (Object child : node.getAllChildren()) {
            if (child instanceof TagNode) {
                collectTableRows((TagNode) child, rows);
            }
        }
    }

    private TagNode findBySelector(TagNode root, String selector) {
        if (selector.startsWith("#")) {
            String id = selector.substring(1);
            return findById(root, id);
        } else if (selector.startsWith(".")) {
            String cls = selector.substring(1);
            return findByClass(root, cls);
        } else {
            TagNode[] found = root.getElementsByName(selector, true);
            return found.length > 0 ? found[0] : null;
        }
    }

    private TagNode findById(TagNode node, String id) {
        String nodeId = node.getAttributeByName("id");
        if (id.equals(nodeId)) return node;
        for (Object child : node.getAllChildren()) {
            if (child instanceof TagNode) {
                TagNode result = findById((TagNode) child, id);
                if (result != null) return result;
            }
        }
        return null;
    }

    private TagNode findByClass(TagNode node, String cls) {
        String classAttr = node.getAttributeByName("class");
        if (classAttr != null && classAttr.contains(cls)) return node;
        for (Object child : node.getAllChildren()) {
            if (child instanceof TagNode) {
                TagNode result = findByClass((TagNode) child, cls);
                if (result != null) return result;
            }
        }
        return null;
    }

    private String cleanupText(String text) {
        text = text.replaceAll("\\n{4,}", "\n\n\n");
        String[] lines = text.split("\\n");
        StringBuilder sb = new StringBuilder();
        for (String line : lines) {
            sb.append(line.stripTrailing()).append("\n");
        }
        return sb.toString().strip();
    }

    private String formatAsJson(String text) {
        StringBuilder sb = new StringBuilder();
        sb.append("{\n");
        sb.append("  \"text\": ").append(jsonEscape(text));
        if (config.includeMetadata) {
            sb.append(",\n  \"metadata\": {\n");
            int count = 0;
            for (Map.Entry<String, String> entry : metadata.entrySet()) {
                if (count > 0) sb.append(",\n");
                sb.append("    ").append(jsonEscape(entry.getKey())).append(": ")
                  .append(jsonEscape(entry.getValue()));
                count++;
            }
            sb.append("\n  }");
        }
        sb.append("\n}");
        return sb.toString();
    }

    private String jsonEscape(String s) {
        return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"")
                       .replace("\n", "\\n").replace("\r", "\\r")
                       .replace("\t", "\\t") + "\"";
    }

    private String padRight(String s, int width) {
        if (s.length() >= width) return s;
        return s + " ".repeat(width - s.length());
    }

    public Map<String, String> getMetadata() {
        return metadata;
    }

    public static void main(String[] args) {
        if (args.length < 1) {
            System.out.println("HTML to Plain Text Extractor");
            System.out.println("Usage: java HtmlToTextExtractor <input> [options]");
            System.out.println();
            System.out.println("Options:");
            System.out.println("  --output <file>       Write output to file");
            System.out.println("  --format <text|json>  Output format (default: text)");
            System.out.println("  --selector <css>      CSS selector to filter content");
            System.out.println("  --no-links            Do not preserve link URLs");
            System.out.println("  --no-tables           Do not format tables");
            System.out.println("  --metadata            Include metadata");
            System.out.println("  --wrap <width>        Line wrap width (default: 80)");
            return;
        }

        String input = args[0];
        ExtractorConfig config = new ExtractorConfig();
        String outputFile = null;

        for (int i = 1; i < args.length; i++) {
            switch (args[i]) {
                case "--output": outputFile = args[++i]; break;
                case "--format": config.outputFormat = args[++i]; break;
                case "--selector": config.cssSelector = args[++i]; break;
                case "--no-links": config.preserveLinks = false; break;
                case "--no-tables": config.preserveTables = false; break;
                case "--metadata": config.includeMetadata = true; break;
                case "--wrap": config.wrapWidth = Integer.parseInt(args[++i]); break;
            }
        }

        HtmlToTextExtractor extractor = new HtmlToTextExtractor(config);

        try {
            String result;
            if (input.startsWith("http://") || input.startsWith("https://")) {
                result = extractor.extractFromUrl(input);
            } else {
                result = extractor.extractFromFile(input);
            }

            if (outputFile != null) {
                FileUtils.writeStringToFile(new File(outputFile), result, StandardCharsets.UTF_8);
                System.out.println("Output written to " + outputFile);
            } else {
                System.out.println(result);
            }
        } catch (Exception e) {
            System.err.println("Error: " + e.getMessage());
            System.exit(1);
        }
    }
}

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.example</groupId>
    <artifactId>html-to-text-extractor</artifactId>
    <version>1.0.0</version>
    <packaging>jar</packaging>

    <name>HTML to Plain Text Extractor</name>
    <description>Converts HTML to clean plain text preserving formatting, tables, lists, links with CSS selectors</description>

    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>net.sourceforge.htmlcleaner</groupId>
            <artifactId>htmlcleaner</artifactId>
            <version>2.29</version>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.15.1</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>3.3.0</version>
                <configuration>
                    <archive>
                        <manifest>
                            <mainClass>HtmlToTextExtractor</mainClass>
                        </manifest>
                    </archive>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>

# HTML to Plain Text Extractor

Converts HTML to clean plain text preserving formatting, tables, lists, and links with CSS selector support.

## Dependencies

- **htmlcleaner** (2.29) - Java HTML parser that cleans and normalizes malformed HTML
- **commons-io** (2.15.1) - Apache Commons IO for file reading and writing utilities

## Build

```bash
mvn clean package
```

## Usage

```bash
java -jar target/html-to-text-extractor-1.0.0.jar <input> [options]
```

### Options

| Option              | Description                                |
|---------------------|--------------------------------------------|
| `--output <file>`   | Write output to a file                     |
| `--format <type>`   | Output format: text or json                |
| `--selector <css>`  | CSS selector to filter content             |
| `--no-links`        | Do not preserve link URLs                  |
| `--no-tables`       | Do not format tables                       |
| `--metadata`        | Include document metadata                  |
| `--wrap <width>`    | Line wrap width (default: 80)              |

## Examples

```bash
java -jar target/html-to-text-extractor-1.0.0.jar page.html
java -jar target/html-to-text-extractor-1.0.0.jar https://example.com --format json
java -jar target/html-to-text-extractor-1.0.0.jar page.html --selector "#content" -o output.txt
```