HTTP REST Client with Retry Logic (javascript, written by Codex)

Codex wrote this javascript project from the task below. It does not run on a clean Ubuntu 22.04 machine as written.

Task given to the agent:

TASK: HTTP REST Client with Retry Logic

Write a program that sends HTTP requests to REST APIs with configurable retry logic, response validation, request chaining, and comprehensive logging of request/response cycles.

FUNCTIONAL REQUIREMENTS:
- Accept a URL as a command-line argument with --method flag (GET, POST, PUT, PATCH, DELETE; default: GET)
- Support sending request body via --data flag (JSON string) or --file flag (read body from a file)
- Support custom headers via --header flag (repeatable, e.g., --header "Content-Type: application/json" --header "Authorization: Bearer token")
- Implement configurable retry logic: --retries flag (default 3), --backoff flag for backoff strategy (linear, exponential; default: exponential), --retry-delay flag for initial delay in seconds (default 1)
- Retry only on configurable HTTP status codes via --retry-on flag (default: 429, 500, 502, 503, 504)
- Support request timeout via --timeout flag (default 30 seconds)
- Support request chaining from a JSON recipe file via --chain flag: execute a sequence of requests where later requests can reference values from earlier responses using JSONPath expressions
- Log complete request/response cycles: method, URL, headers, body, status code, response headers, response body, timing, and retry attempts
- Support response validation via --expect-status flag (e.g., --expect-status 200) and --expect-json flag (JSONPath expression that must exist in the response)
- Print formatted response to console: status code, headers, and pretty-printed body (auto-detect JSON and format it)
- Save the full request/response log as JSON with --output flag (default: request_log.json)
- If no URL is given, demonstrate the client by sending requests to a public test API (httpbin.org or similar), showing GET, POST with JSON body, retry on simulated failure, and a simple request chain
- Handle errors: connection refused, DNS failure, SSL errors, malformed JSON responses, and redirect loops

Create a complete JavaScript project for a clean Ubuntu 22.04 machine with only Node.js 20+ (LTS) installed. Include:
- Source code
- package.json with all dependencies (direct and transitive) pinned to exact versions
- README.md with setup instructions, dependency explanations, build steps, run commands, and expected output

[]

{
  "name": "http-rest-client-retry",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "http-rest-client-retry",
      "version": "1.0.0",
      "dependencies": {
        "got": "14.4.3",
        "jsonpath": "1.1.1"
      },
      "engines": {
        "node": ">=20.0.0"
      }
    },
    "node_modules/got": {
      "version": "14.4.3"
    },
    "node_modules/jsonpath": {
      "version": "1.1.1"
    }
  }
}

{
  "name": "http-rest-client-retry",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "main": "src/index.js",
  "scripts": {
    "start": "node src/index.js"
  },
  "engines": {
    "node": ">=20.0.0"
  },
  "dependencies": {
    "got": "14.4.3",
    "jsonpath": "1.1.1"
  }
}

# HTTP REST Client with Retry Logic (JavaScript)

Sends REST requests with configurable retry logic, backoff strategy, timeout, status-based retry filters, JSONPath validation, request chaining, and full request/response logging.

## Requirements

- Ubuntu 22.04
- Node.js 20+ (LTS)

## Dependencies (Pinned)

- `got@14.4.3`
- `jsonpath@1.1.1`

## Setup

```bash
npm install
```

## Run

```bash
node src/index.js https://httpbin.org/get
node src/index.js https://httpbin.org/post --method POST --data '{"hello":"world"}' --header "Content-Type: application/json"
node src/index.js https://httpbin.org/status/503 --retries 3 --backoff exponential --retry-delay 1 --retry-on 503
node src/index.js https://httpbin.org/get --expect-status 200 --expect-json '$.url'
node src/index.js --chain chain.json
node src/index.js
```

## Notes

- `--header` is repeatable.
- `--data` accepts inline JSON text; `--file` reads body from file.
- `--chain` expects a JSON array of request objects.
- No URL launches a built-in demo against `httpbin.org` including chain and retry behavior.

import fs from "fs";
import path from "path";
import got from "got";
import jp from "jsonpath";

const DEMO_CHAIN = [
  { method: "GET", url: "https://httpbin.org/get", expect_status: 200, expect_json: "$.url" },
  { method: "POST", url: "https://httpbin.org/post", data: { message: "hello" }, expect_status: 200, expect_json: "$.json.message" },
  { method: "GET", url: "https://httpbin.org/status/503", retries: 2, retry_on: [503], expect_status: 503 },
  { method: "GET", url: "https://httpbin.org/anything/{{step1.$.json.message}}", expect_status: 200 },
];

function parseArgs(argv) {
  const options = {};
  const positional = [];
  for (let i = 0; i < argv.length; i += 1) {
    const t = argv[i];
    if (t.startsWith("--")) {
      const key = t.slice(2);
      const next = argv[i + 1];
      const value = next && !next.startsWith("--") ? next : true;
      if (value !== true) i += 1;
      if (key === "header") {
        if (!options.header) options.header = [];
        options.header.push(String(value));
      } else options[key] = value;
    } else positional.push(t);
  }
  return { options, positional };
}

function parseHeaders(raw = []) {
  const out = {};
  for (const h of raw) {
    const idx = h.indexOf(":");
    if (idx <= 0) continue;
    out[h.slice(0, idx).trim()] = h.slice(idx + 1).trim();
  }
  return out;
}

function parseRetryOn(raw) {
  const text = String(raw || "429,500,502,503,504");
  return new Set(text.split(",").map((x) => Number.parseInt(x.trim(), 10)).filter(Number.isFinite));
}

function delay(ms) {
  return new Promise((r) => setTimeout(r, ms));
}

function safeJsonParse(s) {
  try {
    return { ok: true, value: JSON.parse(s) };
  } catch {
    return { ok: false, value: null };
  }
}

function replaceRefs(text, context) {
  return String(text).replace(/\{\{step(\d+)\.(\$[^}]+)\}\}/g, (_, idxRaw, expr) => {
    const idx = Number.parseInt(idxRaw, 10);
    const body = context[idx]?.response_json;
    if (!body) return "";
    const vals = jp.query(body, expr);
    return vals.length ? String(vals[0]) : "";
  });
}

async function requestWithRetry(spec, defaults, context, log) {
  const method = String(spec.method || defaults.method || "GET").toUpperCase();
  const retryOn = new Set(spec.retry_on || [...defaults.retryOn]);
  const retries = Number(spec.retries ?? defaults.retries);
  const timeoutMs = Number(spec.timeout ?? defaults.timeout) * 1000;
  const backoff = String(spec.backoff || defaults.backoff);
  const retryDelayMs = Number(spec.retryDelay ?? defaults.retryDelay) * 1000;

  const headers = { ...defaults.headers, ...(spec.headers || {}) };
  const url = replaceRefs(spec.url, context);
  const dataObj = spec.data != null ? spec.data : null;
  const bodyRaw = dataObj == null ? null : (typeof dataObj === "string" ? replaceRefs(dataObj, context) : JSON.stringify(dataObj));

  let attempt = 0;
  while (true) {
    const started = Date.now();
    try {
      const res = await got(url, {
        method,
        headers,
        body: bodyRaw,
        throwHttpErrors: false,
        retry: { limit: 0 },
        timeout: { request: timeoutMs },
        followRedirect: true,
        maxRedirects: 10,
      });
      const elapsed = Date.now() - started;
      const parsed = safeJsonParse(res.body);
      const entry = {
        attempt,
        method,
        url,
        request_headers: headers,
        request_body: bodyRaw,
        status_code: res.statusCode,
        response_headers: res.headers,
        response_body: res.body,
        response_json: parsed.ok ? parsed.value : null,
        timing_ms: elapsed,
      };
      log.push(entry);

      if (retryOn.has(res.statusCode) && attempt < retries) {
        attempt += 1;
        const d = backoff === "linear" ? retryDelayMs * attempt : retryDelayMs * (2 ** (attempt - 1));
        await delay(d);
        continue;
      }

      if (spec.expect_status != null && Number(spec.expect_status) !== res.statusCode) {
        throw new Error(`Expected status ${spec.expect_status}, got ${res.statusCode}`);
      }
      if (spec.expect_json && parsed.ok) {
        const vals = jp.query(parsed.value, String(spec.expect_json));
        if (!vals.length) throw new Error(`JSONPath not found: ${spec.expect_json}`);
      } else if (spec.expect_json && !parsed.ok) {
        throw new Error("Expected JSON response but body is not JSON");
      }

      return entry;
    } catch (e) {
      const elapsed = Date.now() - started;
      log.push({
        attempt,
        method,
        url,
        request_headers: headers,
        request_body: bodyRaw,
        status_code: 0,
        response_headers: {},
        response_body: "",
        response_json: null,
        timing_ms: elapsed,
        error: e instanceof Error ? e.message : String(e),
      });
      if (attempt >= retries) throw e;
      attempt += 1;
      const d = backoff === "linear" ? retryDelayMs * attempt : retryDelayMs * (2 ** (attempt - 1));
      await delay(d);
    }
  }
}

function printResponse(entry) {
  console.log(`Status: ${entry.status_code}`);
  console.log("Headers:");
  for (const [k, v] of Object.entries(entry.response_headers || {})) console.log(`  ${k}: ${v}`);
  console.log("Body:");
  if (entry.response_json) console.log(JSON.stringify(entry.response_json, null, 2));
  else console.log(entry.response_body || "");
}

async function runChain(chain, defaults, outputPath) {
  const logs = [];
  const context = [];
  for (const spec of chain) {
    const log = [];
    const result = await requestWithRetry(spec, defaults, context, log);
    logs.push({ request: spec, attempts: log });
    context.push(result);
    printResponse(result);
  }
  fs.writeFileSync(outputPath, `${JSON.stringify({ generated_at: new Date().toISOString(), chain, logs }, null, 2)}\n`, "utf8");
  console.log(`Saved log: ${outputPath}`);
}

async function main() {
  const { options, positional } = parseArgs(process.argv.slice(2));
  const url = positional[0] || "";
  const defaults = {
    method: String(options.method || "GET"),
    headers: parseHeaders(options.header || []),
    retries: Number(options.retries ?? 3),
    backoff: String(options.backoff || "exponential"),
    retryDelay: Number(options["retry-delay"] ?? 1),
    retryOn: parseRetryOn(options["retry-on"]),
    timeout: Number(options.timeout ?? 30),
  };
  const output = path.resolve(String(options.output || "request_log.json"));

  if (options.chain) {
    const chain = JSON.parse(fs.readFileSync(path.resolve(String(options.chain)), "utf8"));
    await runChain(Array.isArray(chain) ? chain : chain.requests, defaults, output);
    return;
  }

  if (!url) {
    await runChain(DEMO_CHAIN, defaults, output);
    return;
  }

  let data = null;
  if (options.file) data = fs.readFileSync(path.resolve(String(options.file)), "utf8");
  else if (options.data) data = String(options.data);
  const spec = {
    method: defaults.method,
    url,
    headers: defaults.headers,
    data,
    expect_status: options["expect-status"] ? Number(options["expect-status"]) : undefined,
    expect_json: options["expect-json"] || undefined,
  };
  const log = [];
  const result = await requestWithRetry(spec, defaults, [], log);
  printResponse(result);
  fs.writeFileSync(output, `${JSON.stringify({ generated_at: new Date().toISOString(), request: spec, attempts: log }, null, 2)}\n`, "utf8");
  console.log(`Saved log: ${output}`);
}

main().catch((e) => {
  console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
  process.exit(1);
});