Robots Check API

POST /v1/robots-check
content-type: application/json
payment-signature: <base64 x402 payload>

{
  "url": "https://grist.tools/v1/whois",
  "user_agent": "GPTBot",
  "include_sitemaps": true,
  "additional_paths": [
    "/docs/whois",
    "/v1/whois/schema"
  ]
}

200 OK
payment-response: <base64 settlement receipt>

{
  "url": "https://grist.tools/v1/whois",
  "source_url": "https://grist.tools/robots.txt",
  "robots_url": "https://grist.tools/robots.txt",
  "robots_status": "found",
  "robots_http_status": 200,
  "user_agent": "GPTBot",
  "user_agent_token": "GPTBot",
  "allowed": false,
  "decision_reason": "rule",
  "matched_rule": {
    "type": "disallow",
    "path": "/v1/",
    "line": 3,
    "octet_length": 4
  },
  "matched_group": {
    "user_agents": [
      "*"
    ],
    "is_wildcard": true,
    "line": 1
  },
  "competing_rules": [
    {
      "type": "disallow",
      "path": "/v1/",
      "line": 3,
      "octet_length": 4
    },
    {
      "type": "allow",
      "path": "/",
      "line": 2,
      "octet_length": 1
    }
  ],
  "competing_rules_total": 2,
  "competing_rules_truncated": false,
  "crawl_delay": null,
  "sitemaps": [
    "https://grist.tools/sitemap.xml"
  ],
  "sitemaps_total": 1,
  "sitemaps_truncated": false,
  "groups_found": 1,
  "rules_parsed": 3,
  "unparsed_lines": 0,
  "robots_size_bytes": 141,
  "robots_truncated": false,
  "additional_results": [
    {
      "path": "/docs/whois",
      "allowed": true,
      "decision_reason": "rule",
      "matched_rule": {
        "type": "allow",
        "path": "/",
        "line": 2,
        "octet_length": 1
      }
    },
    {
      "path": "/v1/whois/schema",
      "allowed": true,
      "decision_reason": "rule",
      "matched_rule": {
        "type": "allow",
        "path": "/v1/*/schema",
        "line": 4,
        "octet_length": 12
      }
    }
  ],
  "fetched_at": "2026-09-01T12:00:00.000Z",
  "checked_at": "2026-09-01T12:00:00.000Z"
}

// robots-check: $0.002 per call, paid in USD Coin on eip155:8453 via x402
// Install: npm install @x402/fetch@2 @x402/evm@2 viem@2
// Save as client.mjs (ES module, Node 18+), export EVM_PRIVATE_KEY with the paying wallet's key in your shell, then run: node client.mjs
// The wrapper reads the 402 (payment-required), signs and retries with payment-signature.
import { wrapFetchWithPayment, x402Client, decodePaymentResponseHeader } from '@x402/fetch'
import { ExactEvmScheme } from '@x402/evm/exact/client'
import { privateKeyToAccount } from 'viem/accounts'

const account = privateKeyToAccount(process.env.EVM_PRIVATE_KEY)
const client = new x402Client().register('eip155:8453', new ExactEvmScheme(account))
const fetchWithPayment = wrapFetchWithPayment(fetch, client)

const body = {
  "url": "https://grist.tools/v1/whois",
  "user_agent": "GPTBot",
  "include_sitemaps": true,
  "additional_paths": [
    "/docs/whois",
    "/v1/whois/schema"
  ]
}

const res = await fetchWithPayment('https://grist.tools/v1/robots-check', {
  method: 'POST',
  headers: { 'content-type': 'application/json' },
  body: JSON.stringify(body),
})
console.log(res.status, await res.json())
const settle = res.headers.get('payment-response')
if (settle) console.log(decodePaymentResponseHeader(settle).transaction)

# robots-check: $0.002 per call, paid in USD Coin on eip155:8453 via x402
# Install: pip install "x402[requests,evm]>=2.24,<3"
# The session reads the 402 (payment-required), signs and retries with payment-signature.
import os
from eth_account import Account
from x402 import x402ClientSync
from x402.http import decode_payment_response_header
from x402.http.clients.requests import x402_requests
from x402.mechanisms.evm.exact import register_exact_evm_client
from x402.mechanisms.evm.signers import EthAccountSigner

account = Account.from_key(os.environ["EVM_PRIVATE_KEY"])
client = x402ClientSync()
register_exact_evm_client(client, EthAccountSigner(account), networks="eip155:8453")
session = x402_requests(client)

payload = {
    "url": "https://grist.tools/v1/whois",
    "user_agent": "GPTBot",
    "include_sitemaps": True,
    "additional_paths": [
        "/docs/whois",
        "/v1/whois/schema"
    ]
}

res = session.post("https://grist.tools/v1/robots-check", json=payload)
print(res.status_code, res.json())
settle = res.headers.get("payment-response")
if settle:
    print(decode_payment_response_header(settle).transaction)

# robots-check: $0.002 per call, paid in USD Coin on eip155:8453 via x402
# 1. Unpaid call: HTTP 402, the requirements in the payment-required header (base64 JSON) and in the body.
curl -i -X POST 'https://grist.tools/v1/robots-check' -H 'content-type: application/json' -d '{"url":"https://grist.tools/v1/whois","user_agent":"GPTBot","include_sitemaps":true,"additional_paths":["/docs/whois","/v1/whois/schema"]}'

# 2. Same call with the signed payment (an EIP-3009 authorization, EIP-712 signed: it cannot be
#    typed by hand). x-payment is accepted as the v1 alternative. HTTP 200 carries payment-response.
curl -i -X POST 'https://grist.tools/v1/robots-check' -H 'content-type: application/json' -H 'payment-signature: <base64 x402 payload>' -d '{"url":"https://grist.tools/v1/whois","user_agent":"GPTBot","include_sitemaps":true,"additional_paths":["/docs/whois","/v1/whois/schema"]}'