{
  "name": "sitemap-parse",
  "summary": "Parses a sitemap, a sitemap index or a bare origin into a flat, bounded list of URLs with lastmod, changefreq and priority.",
  "rationale": "Needs the network, follows a sitemap index across many documents and normalises four different legal sitemap formats. An agent that does it itself either writes an XML parser with the XXE and fan-out defences, or fetches an unbounded tree of documents a stranger chose.",
  "family": "feeds",
  "price_usd": "0.002",
  "free": false,
  "timeout_ms": 14000,
  "max_timeout_seconds": 35,
  "input_schema": {
    "$schema": "https://json-schema.org/draft/2020-12/schema",
    "type": "object",
    "properties": {
      "url": {
        "type": "string",
        "maxLength": 2048,
        "format": "uri",
        "pattern": "^[hH][tT][tT][pP][sS]?:",
        "description": "Absolute http(s) URL, up to 2048 characters, of a sitemap, a sitemap index or a bare origin (path / and no query); a bare origin is resolved through robots.txt Sitemap: lines or /sitemap.xml."
      },
      "limit": {
        "default": 5000,
        "description": "Maximum number of URLs returned across every document read, 1 to 50000 (default 5000); past it the list is cut and reported as truncated.",
        "type": "integer",
        "minimum": 1,
        "maximum": 50000
      },
      "include_metadata": {
        "default": false,
        "description": "When true each URL carries its lastmod, changefreq and priority; when false (default) those three fields are null.",
        "type": "boolean"
      },
      "max_sitemaps": {
        "default": 10,
        "description": "Maximum number of sitemap documents fetched, entry document included, 1 to 50 (default 10); children past it are reported as skipped with reason sitemap_limit.",
        "type": "integer",
        "minimum": 1,
        "maximum": 50
      },
      "discover_from_robots": {
        "default": true,
        "description": "Only for a bare origin: when true (default) the Sitemap: lines of /robots.txt are read first with /sitemap.xml as the fallback, when false /sitemap.xml is used directly.",
        "type": "boolean"
      }
    },
    "required": [
      "url"
    ],
    "additionalProperties": false
  },
  "input_example": {
    "url": "https://grist.tools/samples/web/fieldnotes/sitemap-index.xml",
    "limit": 100,
    "include_metadata": true
  },
  "output_example": {
    "requested_url": "https://grist.tools/samples/web/fieldnotes/sitemap-index.xml",
    "entry_url": "https://grist.tools/samples/web/fieldnotes/sitemap-index.xml",
    "source_url": "https://grist.tools/samples/web/fieldnotes/sitemap-index.xml",
    "discovery": "direct",
    "kind": "mixed",
    "format": "xml",
    "urls": [
      {
        "loc": "https://grist.tools/samples/web/fieldnotes/index.html",
        "lastmod": "2026-08-25T00:00:00Z",
        "changefreq": "daily",
        "priority": 1
      },
      {
        "loc": "https://grist.tools/samples/web/fieldnotes/about.html",
        "lastmod": "2026-08-20T00:00:00Z",
        "changefreq": "yearly",
        "priority": 0.3
      },
      {
        "loc": "https://grist.tools/samples/web/fieldnotes/notes/first-frost.html",
        "lastmod": "2026-08-24T06:30:00Z",
        "changefreq": "monthly",
        "priority": 0.6
      },
      {
        "loc": "https://grist.tools/samples/web/fieldnotes/notes/rain-gauge.html",
        "lastmod": "2026-08-25T06:15:00Z",
        "changefreq": null,
        "priority": 0.6
      }
    ],
    "count": 4,
    "total_seen": 4,
    "truncated": false,
    "stopped_reason": null,
    "sitemap_count": 3,
    "sitemaps_visited": [
      {
        "url": "https://grist.tools/samples/web/fieldnotes/sitemap-index.xml",
        "kind": "sitemapindex",
        "url_count": 0,
        "http_status": 200,
        "bytes": 407,
        "gzip": false,
        "depth": 0
      },
      {
        "url": "https://grist.tools/samples/web/fieldnotes/sitemap-pages.xml",
        "kind": "urlset",
        "url_count": 2,
        "http_status": 200,
        "bytes": 479,
        "gzip": false,
        "depth": 1
      },
      {
        "url": "https://grist.tools/samples/web/fieldnotes/sitemap-notes.xml",
        "kind": "urlset",
        "url_count": 2,
        "http_status": 200,
        "bytes": 493,
        "gzip": false,
        "depth": 1
      }
    ],
    "sitemaps_skipped": [],
    "sitemaps_skipped_total": 0,
    "sitemaps_skipped_by_reason": {
      "depth_limit": 0,
      "sitemap_limit": 0,
      "cross_host": 0,
      "too_large": 0,
      "malformed": 0,
      "duplicate": 0,
      "time_budget": 0,
      "not_fetchable": 0
    },
    "invalid_loc_count": 0,
    "invalid_lastmod_count": 0,
    "robots_sitemaps": null,
    "fetched_at": "2026-09-01T12:00:00.000Z"
  },
  "errors": [
    "invalid_input",
    "blocked_target",
    "unreachable_target",
    "upstream_status",
    "upstream_timeout",
    "unsupported_content_type",
    "too_large",
    "unprocessable",
    "internal"
  ]
}