HTML to Plain Text Extractor (java, written by Claude Code)

--- /dev/null
+++ b/src/main/java/HtmlExtractor.java
@@ -0,0 +1,494 @@
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.*;
+import org.jsoup.select.Elements;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonArray;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * HTML to Plain Text Extractor.
+ *
+ * Converts HTML documents to clean plain text preserving formatting,
+ * tables, lists, and links. Supports CSS selectors for targeted extraction.
+ * Can output results as plain text or structured JSON.
+ */
+public class HtmlExtractor {
+
+    private String baseUrl;
+    private int lineWidth;
+    private boolean jsonOutput;
+
+    public HtmlExtractor() {
+        this(null, 80, false);
+    }
+
+    public HtmlExtractor(String baseUrl, int lineWidth, boolean jsonOutput) {
+        this.baseUrl = baseUrl;
+        this.lineWidth = lineWidth;
+        this.jsonOutput = jsonOutput;
+    }
+
+    /**
+     * Extract plain text from an HTML string.
+     *
+     * @param html         The HTML content to convert.
+     * @param cssSelector  Optional CSS selector for targeted extraction (may be null).
+     * @return Clean plain text representation.
+     */
+    public String extract(String html, String cssSelector) {
+        Document doc = Jsoup.parse(html);
+
+        if (baseUrl != null && !baseUrl.isEmpty()) {
+            doc.setBaseUri(baseUrl);
+        }
+
+        Elements targetElements;
+        if (cssSelector != null && !cssSelector.isEmpty()) {
+            targetElements = doc.select(cssSelector);
+        } else {
+            // Remove script, style, and head elements
+            doc.select("script, style, head").remove();
+            targetElements = doc.select("body");
+            if (targetElements.isEmpty()) {
+                targetElements = new Elements(doc);
+            }
+        }
+
+        List<String> lines = new ArrayList<>();
+        for (Element element : targetElements) {
+            processNode(element, lines, 0);
+        }
+
+        String result = String.join("\n", lines);
+
+        // Clean up excessive blank lines
+        while (result.contains("\n\n\n")) {
+            result = result.replace("\n\n\n", "\n\n");
+        }
+
+        return result.trim();
+    }
+
+    /**
+     * Extract text and return as structured JSON using Gson.
+     *
+     * @param html         The HTML content.
+     * @param cssSelector  Optional CSS selector.
+     * @return JSON string with extraction results.
+     */
+    public String extractAsJson(String html, String cssSelector) {
+        String plainText = extract(html, cssSelector);
+
+        Document doc = Jsoup.parse(html);
+        if (baseUrl != null) {
+            doc.setBaseUri(baseUrl);
+        }
+
+        JsonObject result = new JsonObject();
+        result.addProperty("text", plainText);
+
+        // Extract metadata
+        JsonObject metadata = new JsonObject();
+        Element titleEl = doc.selectFirst("title");
+        metadata.addProperty("title", titleEl != null ? titleEl.text() : "");
+
+        // Extract all links
+        Elements links;
+        if (cssSelector != null && !cssSelector.isEmpty()) {
+            links = doc.select(cssSelector + " a[href]");
+        } else {
+            links = doc.select("a[href]");
+        }
+
+        JsonArray linksArray = new JsonArray();
+        for (Element link : links) {
+            JsonObject linkObj = new JsonObject();
+            linkObj.addProperty("text", link.text());
+            linkObj.addProperty("href", link.absUrl("href").isEmpty() ? link.attr("href") : link.absUrl("href"));
+            linksArray.add(linkObj);
+        }
+        metadata.add("links", linksArray);
+
+        // Extract headings
+        JsonArray headingsArray = new JsonArray();
+        Elements headings;
+        if (cssSelector != null && !cssSelector.isEmpty()) {
+            headings = doc.select(cssSelector + " h1, " + cssSelector + " h2, " +
+                    cssSelector + " h3, " + cssSelector + " h4, " +
+                    cssSelector + " h5, " + cssSelector + " h6");
+        } else {
+            headings = doc.select("h1, h2, h3, h4, h5, h6");
+        }
+        for (Element h : headings) {
+            JsonObject hObj = new JsonObject();
+            hObj.addProperty("level", Integer.parseInt(h.tagName().substring(1)));
+            hObj.addProperty("text", h.text());
+            headingsArray.add(hObj);
+        }
+        metadata.add("headings", headingsArray);
+
+        result.add("metadata", metadata);
+
+        Gson gson = new GsonBuilder().setPrettyPrinting().create();
+        return gson.toJson(result);
+    }
+
+    private void processNode(Node node, List<String> lines, int indent) {
+        if (node instanceof TextNode) {
+            String text = ((TextNode) node).text().trim();
+            if (!text.isEmpty()) {
+                lines.add(wrapText(text, indent));
+            }
+            return;
+        }
+
+        if (!(node instanceof Element)) {
+            return;
+        }
+
+        Element element = (Element) node;
+        String tagName = element.tagName().toLowerCase();
+
+        switch (tagName) {
+            case "h1": case "h2": case "h3":
+            case "h4": case "h5": case "h6":
+                processHeading(element, lines, indent);
+                break;
+            case "p":
+                lines.add("");
+                processChildren(element, lines, indent);
+                lines.add("");
+                break;
+            case "br":
+                lines.add("");
+                break;
+            case "hr":
+                lines.add("");
+                lines.add(repeat("-", Math.min(lineWidth, 40)));
+                lines.add("");
+                break;
+            case "ul":
+                processUnorderedList(element, lines, indent);
+                break;
+            case "ol":
+                processOrderedList(element, lines, indent);
+                break;
+            case "table":
+                processTable(element, lines, indent);
+                break;
+            case "a":
+                processLink(element, lines);
+                break;
+            case "blockquote":
+                processBlockquote(element, lines, indent);
+                break;
+            case "pre":
+                processPreformatted(element, lines, indent);
+                break;
+            case "code":
+                if (!element.parent().tagName().equalsIgnoreCase("pre")) {
+                    lines.add("`" + element.text() + "`");
+                } else {
+                    processChildren(element, lines, indent);
+                }
+                break;
+            case "strong": case "b":
+                lines.add("**" + element.text() + "**");
+                break;
+            case "em": case "i":
+                lines.add("_" + element.text() + "_");
+                break;
+            case "dl":
+                processDefinitionList(element, lines, indent);
+                break;
+            default:
+                processChildren(element, lines, indent);
+                break;
+        }
+    }
+
+    private void processHeading(Element element, List<String> lines, int indent) {
+        int level = Integer.parseInt(element.tagName().substring(1));
+        String text = element.text().trim();
+        String prefix = repeat("#", level) + " ";
+        lines.add("");
+        lines.add(prefix + text);
+        lines.add(level <= 2 ? repeat("=", text.length()) : repeat("-", text.length()));
+        lines.add("");
+    }
+
+    private void processUnorderedList(Element element, List<String> lines, int indent) {
+        lines.add("");
+        for (Element li : element.children()) {
+            if (li.tagName().equalsIgnoreCase("li")) {
+                String prefix = repeat(" ", indent) + "  * ";
+                String text = li.text().trim();
+                Element link = li.selectFirst("a[href]");
+                if (link != null) {
+                    String href = link.absUrl("href").isEmpty() ? link.attr("href") : link.absUrl("href");
+                    text += " [" + href + "]";
+                }
+                lines.add(prefix + text);
+            }
+        }
+        lines.add("");
+    }
+
+    private void processOrderedList(Element element, List<String> lines, int indent) {
+        lines.add("");
+        int start = 1;
+        String startAttr = element.attr("start");
+        if (!startAttr.isEmpty()) {
+            try {
+                start = Integer.parseInt(startAttr);
+            } catch (NumberFormatException ignored) {
+            }
+        }
+        int idx = start;
+        for (Element li : element.children()) {
+            if (li.tagName().equalsIgnoreCase("li")) {
+                String prefix = repeat(" ", indent) + "  " + idx + ". ";
+                String text = li.text().trim();
+                Element link = li.selectFirst("a[href]");
+                if (link != null) {
+                    String href = link.absUrl("href").isEmpty() ? link.attr("href") : link.absUrl("href");
+                    text += " [" + href + "]";
+                }
+                lines.add(prefix + text);
+                idx++;
+            }
+        }
+        lines.add("");
+    }
+
+    private void processTable(Element table, List<String> lines, int indent) {
+        List<List<String>> rows = new ArrayList<>();
+        for (Element tr : table.select("tr")) {
+            List<String> cells = new ArrayList<>();
+            for (Element cell : tr.select("th, td")) {
+                cells.add(cell.text().trim());
+            }
+            if (!cells.isEmpty()) {
+                rows.add(cells);
+            }
+        }
+
+        if (rows.isEmpty()) return;
+
+        int maxCols = 0;
+        for (List<String> row : rows) {
+            maxCols = Math.max(maxCols, row.size());
+        }
+
+        int[] colWidths = new int[maxCols];
+        for (List<String> row : rows) {
+            for (int i = 0; i < row.size(); i++) {
+                colWidths[i] = Math.max(colWidths[i], row.get(i).length());
+            }
+        }
+        for (int i = 0; i < colWidths.length; i++) {
+            colWidths[i] = Math.max(colWidths[i], 3);
+        }
+
+        String prefix = repeat(" ", indent);
+        StringBuilder sep = new StringBuilder(prefix + "+");
+        for (int w : colWidths) {
+            sep.append(repeat("-", w + 2)).append("+");
+        }
+
+        lines.add("");
+        lines.add(sep.toString());
+
+        for (int rowIdx = 0; rowIdx < rows.size(); rowIdx++) {
+            List<String> row = rows.get(rowIdx);
+            while (row.size() < maxCols) {
+                row.add("");
+            }
+            StringBuilder rowStr = new StringBuilder(prefix + "|");
+            for (int i = 0; i < maxCols; i++) {
+                rowStr.append(String.format(" %-" + colWidths[i] + "s ", row.get(i))).append("|");
+            }
+            lines.add(rowStr.toString());
+
+            if (rowIdx == 0 || rowIdx == rows.size() - 1) {
+                lines.add(sep.toString());
+            }
+        }
+        lines.add("");
+    }
+
+    private void processLink(Element element, List<String> lines) {
+        String text = element.text().trim();
+        String href = element.absUrl("href").isEmpty() ? element.attr("href") : element.absUrl("href");
+        if (!href.isEmpty() && !href.equals(text)) {
+            lines.add(text + " [" + href + "]");
+        } else {
+            lines.add(text);
+        }
+    }
+
+    private void processBlockquote(Element element, List<String> lines, int indent) {
+        lines.add("");
+        List<String> innerLines = new ArrayList<>();
+        processChildren(element, innerLines, indent);
+        for (String line : innerLines) {
+            if (!line.trim().isEmpty()) {
+                lines.add(repeat(" ", indent) + "> " + line.trim());
+            } else {
+                lines.add("");
+            }
+        }
+        lines.add("");
+    }
+
+    private void processPreformatted(Element element, List<String> lines, int indent) {
+        lines.add("");
+        lines.add("---");
+        String preText = element.wholeText();
+        for (String line : preText.split("\n")) {
+            lines.add(repeat(" ", indent) + "  " + line);
+        }
+        lines.add("---");
+        lines.add("");
+    }
+
+    private void processDefinitionList(Element element, List<String> lines, int indent) {
+        lines.add("");
+        for (Element child : element.children()) {
+            if (child.tagName().equalsIgnoreCase("dt")) {
+                lines.add(child.text().trim() + ":");
+            } else if (child.tagName().equalsIgnoreCase("dd")) {
+                lines.add("    " + child.text().trim());
+            }
+        }
+        lines.add("");
+    }
+
+    private void processChildren(Element element, List<String> lines, int indent) {
+        for (Node child : element.childNodes()) {
+            processNode(child, lines, indent);
+        }
+    }
+
+    private String wrapText(String text, int indent) {
+        String prefix = repeat(" ", indent);
+        String[] words = text.split("\\s+");
+        if (words.length == 0) return "";
+
+        StringBuilder result = new StringBuilder();
+        StringBuilder currentLine = new StringBuilder(prefix + words[0]);
+
+        for (int i = 1; i < words.length; i++) {
+            if (currentLine.length() + 1 + words[i].length() <= lineWidth) {
+                currentLine.append(" ").append(words[i]);
+            } else {
+                result.append(currentLine).append("\n");
+                currentLine = new StringBuilder(prefix + words[i]);
+            }
+        }
+        result.append(currentLine);
+        return result.toString();
+    }
+
+    private static String repeat(String s, int count) {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < count; i++) {
+            sb.append(s);
+        }
+        return sb.toString();
+    }
+
+    // --- Main CLI ---
+
+    public static void main(String[] args) {
+        String inputPath = null;
+        String outputPath = null;
+        String selector = null;
+        String baseUrl = null;
+        int width = 80;
+        boolean json = false;
+
+        for (int i = 0; i < args.length; i++) {
+            switch (args[i]) {
+                case "-o": case "--output":
+                    outputPath = args[++i];
+                    break;
+                case "-s": case "--selector":
+                    selector = args[++i];
+                    break;
+                case "-b": case "--base-url":
+                    baseUrl = args[++i];
+                    break;
+                case "-w": case "--width":
+                    width = Integer.parseInt(args[++i]);
+                    break;
+                case "-j": case "--json":
+                    json = true;
+                    break;
+                case "-h": case "--help":
+                    printUsage();
+                    return;
+                default:
+                    if (!args[i].startsWith("-")) {
+                        inputPath = args[i];
+                    }
+                    break;
+            }
+        }
+
+        try {
+            String html;
+            if (inputPath != null) {
+                html = new String(Files.readAllBytes(Paths.get(inputPath)), "UTF-8");
+            } else {
+                BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
+                StringBuilder sb = new StringBuilder();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    sb.append(line).append("\n");
+                }
+                html = sb.toString();
+            }
+
+            HtmlExtractor extractor = new HtmlExtractor(baseUrl, width, json);
+            String result;
+            if (json) {
+                result = extractor.extractAsJson(html, selector);
+            } else {
+                result = extractor.extract(html, selector);
+            }
+
+            if (outputPath != null) {
+                Files.write(Paths.get(outputPath), result.getBytes("UTF-8"));
+                System.err.println("Output written to " + outputPath);
+            } else {
+                System.out.println(result);
+            }
+
+        } catch (IOException e) {
+            System.err.println("Error: " + e.getMessage());
+            System.exit(1);
+        }
+    }
+
+    private static void printUsage() {
+        System.out.println("HTML to Plain Text Extractor");
+        System.out.println("Usage: java HtmlExtractor [options] [input.html]");
+        System.out.println();
+        System.out.println("Options:");
+        System.out.println("  -o, --output <file>     Output file path (default: stdout)");
+        System.out.println("  -s, --selector <css>    CSS selector for targeted extraction");
+        System.out.println("  -b, --base-url <url>    Base URL for resolving relative links");
+        System.out.println("  -w, --width <num>       Maximum line width (default: 80)");
+        System.out.println("  -j, --json              Output as structured JSON");
+        System.out.println("  -h, --help              Show this help message");
+    }
+}

Claude Code wrote this java project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTML to Plain Text Extractor

Write a program that converts HTML documents to clean plain text, intelligently handling formatting, tables, lists, and links while removing all markup and scripts.

FUNCTIONAL REQUIREMENTS:
- Accept an HTML file path as a command-line argument
- Strip all HTML tags, CSS styles, JavaScript, and comments while preserving readable text content
- Convert HTML formatting to plain text equivalents: headings become UPPERCASE with underlines, bold text is wrapped in *asterisks*, lists become indented with bullets (- ) or numbers (1.), horizontal rules become dashed lines
- Convert HTML tables to aligned plain text tables with column padding and separator rows
- Convert hyperlinks to "text [URL]" format, or optionally strip URLs via --no-urls flag
- Preserve paragraph spacing: consecutive block elements get blank line separators
- Handle HTML entities: decode & < >   — etc. to their text equivalents
- Support extracting text from only specific HTML elements via --selector flag (CSS selector syntax, e.g., --selector "article" or --selector ".content")
- Support extracting and listing all URLs found in the document via --extract-urls flag
- Set maximum line width via --width flag (default: 80 characters) with word wrapping
- Support batch conversion of multiple HTML files via --batch flag
- Print the plain text output to console by default
- Save to a file via --output flag (default: same base name with .txt extension)
- If no input is given, generate a sample HTML page with headings, paragraphs, links, tables, lists, images, inline styles, scripts, and HTML entities, then convert it and display both the original HTML and the extracted text
- Handle errors: malformed HTML (parse gracefully), encoding detection, and binary file detection

Create a complete Java project for a clean Ubuntu 22.04 machine with only JDK 17+ installed. Include:
- Source code
- pom.xml with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[
  {
    "category": "misspecification",
    "rule": "signature.build_layout_mismatch",
    "source": "failure_signature",
    "excerpt": "error: no classes were compiled"
  }
]

import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.Elements;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.JsonArray;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/**
 * HTML to Plain Text Extractor.
 *
 * Converts HTML documents to clean plain text preserving formatting,
 * tables, lists, and links. Supports CSS selectors for targeted extraction.
 * Can output results as plain text or structured JSON.
 */
public class HtmlExtractor {

    private String baseUrl;
    private int lineWidth;
    private boolean jsonOutput;

    public HtmlExtractor() {
        this(null, 80, false);
    }

    public HtmlExtractor(String baseUrl, int lineWidth, boolean jsonOutput) {
        this.baseUrl = baseUrl;
        this.lineWidth = lineWidth;
        this.jsonOutput = jsonOutput;
    }

    /**
     * Extract plain text from an HTML string.
     *
     * @param html         The HTML content to convert.
     * @param cssSelector  Optional CSS selector for targeted extraction (may be null).
     * @return Clean plain text representation.
     */
    public String extract(String html, String cssSelector) {
        Document doc = Jsoup.parse(html);

        if (baseUrl != null && !baseUrl.isEmpty()) {
            doc.setBaseUri(baseUrl);
        }

        Elements targetElements;
        if (cssSelector != null && !cssSelector.isEmpty()) {
            targetElements = doc.select(cssSelector);
        } else {
            // Remove script, style, and head elements
            doc.select("script, style, head").remove();
            targetElements = doc.select("body");
            if (targetElements.isEmpty()) {
                targetElements = new Elements(doc);
            }
        }

        List<String> lines = new ArrayList<>();
        for (Element element : targetElements) {
            processNode(element, lines, 0);
        }

        String result = String.join("\n", lines);

        // Clean up excessive blank lines
        while (result.contains("\n\n\n")) {
            result = result.replace("\n\n\n", "\n\n");
        }

        return result.trim();
    }

    /**
     * Extract text and return as structured JSON using Gson.
     *
     * @param html         The HTML content.
     * @param cssSelector  Optional CSS selector.
     * @return JSON string with extraction results.
     */
    public String extractAsJson(String html, String cssSelector) {
        String plainText = extract(html, cssSelector);

        Document doc = Jsoup.parse(html);
        if (baseUrl != null) {
            doc.setBaseUri(baseUrl);
        }

        JsonObject result = new JsonObject();
        result.addProperty("text", plainText);

        // Extract metadata
        JsonObject metadata = new JsonObject();
        Element titleEl = doc.selectFirst("title");
        metadata.addProperty("title", titleEl != null ? titleEl.text() : "");

        // Extract all links
        Elements links;
        if (cssSelector != null && !cssSelector.isEmpty()) {
            links = doc.select(cssSelector + " a[href]");
        } else {
            links = doc.select("a[href]");
        }

        JsonArray linksArray = new JsonArray();
        for (Element link : links) {
            JsonObject linkObj = new JsonObject();
            linkObj.addProperty("text", link.text());
            linkObj.addProperty("href", link.absUrl("href").isEmpty() ? link.attr("href") : link.absUrl("href"));
            linksArray.add(linkObj);
        }
        metadata.add("links", linksArray);

        // Extract headings
        JsonArray headingsArray = new JsonArray();
        Elements headings;
        if (cssSelector != null && !cssSelector.isEmpty()) {
            headings = doc.select(cssSelector + " h1, " + cssSelector + " h2, " +
                    cssSelector + " h3, " + cssSelector + " h4, " +
                    cssSelector + " h5, " + cssSelector + " h6");
        } else {
            headings = doc.select("h1, h2, h3, h4, h5, h6");
        }
        for (Element h : headings) {
            JsonObject hObj = new JsonObject();
            hObj.addProperty("level", Integer.parseInt(h.tagName().substring(1)));
            hObj.addProperty("text", h.text());
            headingsArray.add(hObj);
        }
        metadata.add("headings", headingsArray);

        result.add("metadata", metadata);

        Gson gson = new GsonBuilder().setPrettyPrinting().create();
        return gson.toJson(result);
    }

    private void processNode(Node node, List<String> lines, int indent) {
        if (node instanceof TextNode) {
            String text = ((TextNode) node).text().trim();
            if (!text.isEmpty()) {
                lines.add(wrapText(text, indent));
            }
            return;
        }

        if (!(node instanceof Element)) {
            return;
        }

        Element element = (Element) node;
        String tagName = element.tagName().toLowerCase();

        switch (tagName) {
            case "h1": case "h2": case "h3":
            case "h4": case "h5": case "h6":
                processHeading(element, lines, indent);
                break;
            case "p":
                lines.add("");
                processChildren(element, lines, indent);
                lines.add("");
                break;
            case "br":
                lines.add("");
                break;
            case "hr":
                lines.add("");
                lines.add(repeat("-", Math.min(lineWidth, 40)));
                lines.add("");
                break;
            case "ul":
                processUnorderedList(element, lines, indent);
                break;
            case "ol":
                processOrderedList(element, lines, indent);
                break;
            case "table":
                processTable(element, lines, indent);
                break;
            case "a":
                processLink(element, lines);
                break;
            case "blockquote":
                processBlockquote(element, lines, indent);
                break;
            case "pre":
                processPreformatted(element, lines, indent);
                break;
            case "code":
                if (!element.parent().tagName().equalsIgnoreCase("pre")) {
                    lines.add("`" + element.text() + "`");
                } else {
                    processChildren(element, lines, indent);
                }
                break;
            case "strong": case "b":
                lines.add("**" + element.text() + "**");
                break;
            case "em": case "i":
                lines.add("_" + element.text() + "_");
                break;
            case "dl":
                processDefinitionList(element, lines, indent);
                break;
            default:
                processChildren(element, lines, indent);
                break;
        }
    }

    private void processHeading(Element element, List<String> lines, int indent) {
        int level = Integer.parseInt(element.tagName().substring(1));
        String text = element.text().trim();
        String prefix = repeat("#", level) + " ";
        lines.add("");
        lines.add(prefix + text);
        lines.add(level <= 2 ? repeat("=", text.length()) : repeat("-", text.length()));
        lines.add("");
    }

    private void processUnorderedList(Element element, List<String> lines, int indent) {
        lines.add("");
        for (Element li : element.children()) {
            if (li.tagName().equalsIgnoreCase("li")) {
                String prefix = repeat(" ", indent) + "  * ";
                String text = li.text().trim();
                Element link = li.selectFirst("a[href]");
                if (link != null) {
                    String href = link.absUrl("href").isEmpty() ? link.attr("href") : link.absUrl("href");
                    text += " [" + href + "]";
                }
                lines.add(prefix + text);
            }
        }
        lines.add("");
    }

    private void processOrderedList(Element element, List<String> lines, int indent) {
        lines.add("");
        int start = 1;
        String startAttr = element.attr("start");
        if (!startAttr.isEmpty()) {
            try {
                start = Integer.parseInt(startAttr);
            } catch (NumberFormatException ignored) {
            }
        }
        int idx = start;
        for (Element li : element.children()) {
            if (li.tagName().equalsIgnoreCase("li")) {
                String prefix = repeat(" ", indent) + "  " + idx + ". ";
                String text = li.text().trim();
                Element link = li.selectFirst("a[href]");
                if (link != null) {
                    String href = link.absUrl("href").isEmpty() ? link.attr("href") : link.absUrl("href");
                    text += " [" + href + "]";
                }
                lines.add(prefix + text);
                idx++;
            }
        }
        lines.add("");
    }

    private void processTable(Element table, List<String> lines, int indent) {
        List<List<String>> rows = new ArrayList<>();
        for (Element tr : table.select("tr")) {
            List<String> cells = new ArrayList<>();
            for (Element cell : tr.select("th, td")) {
                cells.add(cell.text().trim());
            }
            if (!cells.isEmpty()) {
                rows.add(cells);
            }
        }

        if (rows.isEmpty()) return;

        int maxCols = 0;
        for (List<String> row : rows) {
            maxCols = Math.max(maxCols, row.size());
        }

        int[] colWidths = new int[maxCols];
        for (List<String> row : rows) {
            for (int i = 0; i < row.size(); i++) {
                colWidths[i] = Math.max(colWidths[i], row.get(i).length());
            }
        }
        for (int i = 0; i < colWidths.length; i++) {
            colWidths[i] = Math.max(colWidths[i], 3);
        }

        String prefix = repeat(" ", indent);
        StringBuilder sep = new StringBuilder(prefix + "+");
        for (int w : colWidths) {
            sep.append(repeat("-", w + 2)).append("+");
        }

        lines.add("");
        lines.add(sep.toString());

        for (int rowIdx = 0; rowIdx < rows.size(); rowIdx++) {
            List<String> row = rows.get(rowIdx);
            while (row.size() < maxCols) {
                row.add("");
            }
            StringBuilder rowStr = new StringBuilder(prefix + "|");
            for (int i = 0; i < maxCols; i++) {
                rowStr.append(String.format(" %-" + colWidths[i] + "s ", row.get(i))).append("|");
            }
            lines.add(rowStr.toString());

            if (rowIdx == 0 || rowIdx == rows.size() - 1) {
                lines.add(sep.toString());
            }
        }
        lines.add("");
    }

    private void processLink(Element element, List<String> lines) {
        String text = element.text().trim();
        String href = element.absUrl("href").isEmpty() ? element.attr("href") : element.absUrl("href");
        if (!href.isEmpty() && !href.equals(text)) {
            lines.add(text + " [" + href + "]");
        } else {
            lines.add(text);
        }
    }

    private void processBlockquote(Element element, List<String> lines, int indent) {
        lines.add("");
        List<String> innerLines = new ArrayList<>();
        processChildren(element, innerLines, indent);
        for (String line : innerLines) {
            if (!line.trim().isEmpty()) {
                lines.add(repeat(" ", indent) + "> " + line.trim());
            } else {
                lines.add("");
            }
        }
        lines.add("");
    }

    private void processPreformatted(Element element, List<String> lines, int indent) {
        lines.add("");
        lines.add("---");
        String preText = element.wholeText();
        for (String line : preText.split("\n")) {
            lines.add(repeat(" ", indent) + "  " + line);
        }
        lines.add("---");
        lines.add("");
    }

    private void processDefinitionList(Element element, List<String> lines, int indent) {
        lines.add("");
        for (Element child : element.children()) {
            if (child.tagName().equalsIgnoreCase("dt")) {
                lines.add(child.text().trim() + ":");
            } else if (child.tagName().equalsIgnoreCase("dd")) {
                lines.add("    " + child.text().trim());
            }
        }
        lines.add("");
    }

    private void processChildren(Element element, List<String> lines, int indent) {
        for (Node child : element.childNodes()) {
            processNode(child, lines, indent);
        }
    }

    private String wrapText(String text, int indent) {
        String prefix = repeat(" ", indent);
        String[] words = text.split("\\s+");
        if (words.length == 0) return "";

        StringBuilder result = new StringBuilder();
        StringBuilder currentLine = new StringBuilder(prefix + words[0]);

        for (int i = 1; i < words.length; i++) {
            if (currentLine.length() + 1 + words[i].length() <= lineWidth) {
                currentLine.append(" ").append(words[i]);
            } else {
                result.append(currentLine).append("\n");
                currentLine = new StringBuilder(prefix + words[i]);
            }
        }
        result.append(currentLine);
        return result.toString();
    }

    private static String repeat(String s, int count) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < count; i++) {
            sb.append(s);
        }
        return sb.toString();
    }

    // --- Main CLI ---

    public static void main(String[] args) {
        String inputPath = null;
        String outputPath = null;
        String selector = null;
        String baseUrl = null;
        int width = 80;
        boolean json = false;

        for (int i = 0; i < args.length; i++) {
            switch (args[i]) {
                case "-o": case "--output":
                    outputPath = args[++i];
                    break;
                case "-s": case "--selector":
                    selector = args[++i];
                    break;
                case "-b": case "--base-url":
                    baseUrl = args[++i];
                    break;
                case "-w": case "--width":
                    width = Integer.parseInt(args[++i]);
                    break;
                case "-j": case "--json":
                    json = true;
                    break;
                case "-h": case "--help":
                    printUsage();
                    return;
                default:
                    if (!args[i].startsWith("-")) {
                        inputPath = args[i];
                    }
                    break;
            }
        }

        try {
            String html;
            if (inputPath != null) {
                html = new String(Files.readAllBytes(Paths.get(inputPath)), "UTF-8");
            } else {
                BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
                StringBuilder sb = new StringBuilder();
                String line;
                while ((line = reader.readLine()) != null) {
                    sb.append(line).append("\n");
                }
                html = sb.toString();
            }

            HtmlExtractor extractor = new HtmlExtractor(baseUrl, width, json);
            String result;
            if (json) {
                result = extractor.extractAsJson(html, selector);
            } else {
                result = extractor.extract(html, selector);
            }

            if (outputPath != null) {
                Files.write(Paths.get(outputPath), result.getBytes("UTF-8"));
                System.err.println("Output written to " + outputPath);
            } else {
                System.out.println(result);
            }

        } catch (IOException e) {
            System.err.println("Error: " + e.getMessage());
            System.exit(1);
        }
    }

    private static void printUsage() {
        System.out.println("HTML to Plain Text Extractor");
        System.out.println("Usage: java HtmlExtractor [options] [input.html]");
        System.out.println();
        System.out.println("Options:");
        System.out.println("  -o, --output <file>     Output file path (default: stdout)");
        System.out.println("  -s, --selector <css>    CSS selector for targeted extraction");
        System.out.println("  -b, --base-url <url>    Base URL for resolving relative links");
        System.out.println("  -w, --width <num>       Maximum line width (default: 80)");
        System.out.println("  -j, --json              Output as structured JSON");
        System.out.println("  -h, --help              Show this help message");
    }
}

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.example</groupId>
    <artifactId>html-extractor</artifactId>
    <version>1.0.0</version>
    <packaging>jar</packaging>

    <name>HTML to Plain Text Extractor</name>
    <description>Converts HTML documents to clean plain text preserving formatting, tables, lists, and links.</description>

    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.17.2</version>
        </dependency>
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.10.1</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>3.3.0</version>
                <configuration>
                    <archive>
                        <manifest>
                            <mainClass>HtmlExtractor</mainClass>
                        </manifest>
                    </archive>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.5.1</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

# HTML to Plain Text Extractor (Java - Trial 1)

Converts HTML documents to clean plain text while preserving formatting, tables, lists, and links. Supports CSS selectors for targeted extraction and optional JSON output.

## Dependencies

- **Jsoup** (1.17.2) - HTML parsing, DOM traversal, and CSS selector support
- **Gson** (2.10.1) - Structured JSON output for extraction results

## Build

```bash
mvn clean package
```

## Usage

```bash
# Convert an HTML file
java -jar target/html-extractor-1.0.0.jar input.html

# Read from stdin
cat page.html | java -jar target/html-extractor-1.0.0.jar

# Use CSS selector
java -jar target/html-extractor-1.0.0.jar input.html -s "div.content"

# Output as JSON with metadata
java -jar target/html-extractor-1.0.0.jar input.html -j

# Write to file with base URL
java -jar target/html-extractor-1.0.0.jar input.html -o output.txt -b "https://example.com"
```

## Features

- Preserves headings with markdown-style formatting
- Converts HTML tables to aligned plain text grid tables
- Renders ordered and unordered lists with proper indentation
- Displays link URLs inline
- Handles blockquotes, preformatted text, code blocks
- CSS selector support via Jsoup
- JSON output mode with metadata (title, links, headings) via Gson
- Configurable line width