{"id":"robots-check","title":"robots.txt (RFC 9309) rule reading with llms.txt and AI-use signals","description":"Reads a supplied robots.txt the RFC 9309 way for an agent deciding whether to fetch up to 25 paths as one product-token user agent: group selection (merged groups, \"*\" fallback), longest-match with the Allow-wins-tie rule, \"*\" and \"$\" patterns, percent-encoding normalization, fetch-status handling (4xx = no rules, 5xx = disallow all), the rule and line that decided each path. It also lints a supplied llms.txt (llmstxt.org format) and lists supplied X-Robots-Tag / meta robots / noai / TDM-reservation signals. This is a reading of published rules, not permission: it does NOT fetch the file, interpret terms of service or licences, follow redirects, honour Crawl-delay, or prove the file you supplied is the live one.","roi":"Stops an agent from guessing at robots.txt precedence; every verdict names the deciding line.","audience":"developer","package":"infra-preflight-kit","channels":["x402","mcp"],"endpoint":{"method":"POST","url":"https://agents.writerslogic.com/api/v1/robots-check","path":"/api/v1/robots-check","contentType":"application/json"},"price":{"amount":"0.005","currency":"USDC","network":"base","perCall":true,"failedCallsCharged":false},"inputSchema":{"type":"object","additionalProperties":false,"required":["userAgent","paths"],"properties":{"userAgent":{"type":"string","maxLength":64,"pattern":"^[A-Za-z_-]+$","description":"Product token only (letters, underscore, hyphen), e.g. \"Googlebot\"; no version."},"paths":{"type":"array","minItems":1,"maxItems":25,"items":{"type":"string","maxLength":2048},"description":"Path plus optional query beginning with /, or a full http(s) URL (path and query are used)."},"robotsTxt":{"type":"string","maxLength":512000,"description":"robots.txt body, at most 512,000 UTF-8 bytes (500 KiB)."},"fetchStatus":{"type":"integer","minimum":100,"maximum":599,"description":"HTTP status the robots.txt fetch ended with (after at most 5 redirects). Omit when robotsTxt is supplied and was a 2xx."},"purpose":{"type":"string","enum":["crawl","ai-training"],"description":"ai-training also treats noai, noimageai and tdm-reservation: 1 as restrictions. Default crawl."},"llmsTxt":{"type":"string","maxLength":200000},"headers":{"type":"array","maxItems":50,"items":{"type":"object","required":["name","value"],"additionalProperties":false,"properties":{"name":{"type":"string","maxLength":100},"value":{"type":"string","maxLength":2000}}},"description":"Response headers of the page: X-Robots-Tag, tdm-reservation, tdm-policy."},"metaTags":{"type":"array","maxItems":50,"items":{"type":"object","required":["name","content"],"additionalProperties":false,"properties":{"name":{"type":"string","maxLength":100},"content":{"type":"string","maxLength":2000}}},"description":"HTML meta elements: robots, googlebot, tdm-reservation, tdm-policy."}}},"resultSchema":{"type":"object","properties":{"status":{"type":"string","enum":["pass","fail","unknown","unsupported"],"description":"pass: every rule held. fail: a rule was violated (see witnesses). unknown: the input lacks the evidence to decide. unsupported: outside what this engine covers. Missing evidence is never a pass."},"fileUsed":{"type":"string","enum":["parsed","unavailable-allow-all","unreachable-disallow-all","none"]},"matchedGroup":{"type":"string","enum":["agent","*","none"]},"decisions":{"type":"array","items":{"type":"object","properties":{"path":{"type":"string"},"allowed":{"type":"boolean"},"reason":{"type":"string"},"matched":{"description":"Deciding rule: directive, pattern, line, octets; null when none matched."},"tie":{"type":"boolean","description":"An equal-length rule of the other kind also matched."}}}},"sitemaps":{"type":"array","items":{"type":"string"}},"ignoredRulesBeforeGroup":{"type":"integer"},"llms":{"description":"llms.txt lint: title, summary, sections, link count and problems; null when not supplied."},"signals":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"name":{"type":"string"},"agent":{"description":"User-agent token the directive targets, or null for all."},"token":{"type":"string"},"effect":{"type":"string"}}}},"tdm":{"type":"object","properties":{"reservation":{"type":"string","enum":["reserved","not-reserved","unset"]},"raw":{"description":"Supplied tdm-reservation value."},"policy":{"description":"Supplied tdm-policy URL."},"source":{"description":"header or meta"}}},"note":{"type":"string"},"witnesses":{"type":"array","items":{"type":"object","properties":{"ruleId":{"type":"string"},"detail":{"type":"string"},"path":{"type":"string","description":"JSON pointer or dotted path into the input."},"offset":{"type":"integer"},"index":{"type":"integer"},"expected":{"description":"What the rule required."},"actual":{"description":"What the input had."}}}},"versions":{"type":"object","description":"Every semantic dependency of this answer (engine, tables and their as-of dates, tokenizer, calendar, rounding profile)."},"limits":{"type":"array","items":{"type":"string"}},"deterministic":{"type":"boolean","description":"Always true: pure code, no model."}}},"outputSchema":{"type":"object","properties":{"data":{"type":"object","properties":{"status":{"type":"string","enum":["pass","fail","unknown","unsupported"],"description":"pass: every rule held. fail: a rule was violated (see witnesses). unknown: the input lacks the evidence to decide. unsupported: outside what this engine covers. Missing evidence is never a pass."},"fileUsed":{"type":"string","enum":["parsed","unavailable-allow-all","unreachable-disallow-all","none"]},"matchedGroup":{"type":"string","enum":["agent","*","none"]},"decisions":{"type":"array","items":{"type":"object","properties":{"path":{"type":"string"},"allowed":{"type":"boolean"},"reason":{"type":"string"},"matched":{"description":"Deciding rule: directive, pattern, line, octets; null when none matched."},"tie":{"type":"boolean","description":"An equal-length rule of the other kind also matched."}}}},"sitemaps":{"type":"array","items":{"type":"string"}},"ignoredRulesBeforeGroup":{"type":"integer"},"llms":{"description":"llms.txt lint: title, summary, sections, link count and problems; null when not supplied."},"signals":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"name":{"type":"string"},"agent":{"description":"User-agent token the directive targets, or null for all."},"token":{"type":"string"},"effect":{"type":"string"}}}},"tdm":{"type":"object","properties":{"reservation":{"type":"string","enum":["reserved","not-reserved","unset"]},"raw":{"description":"Supplied tdm-reservation value."},"policy":{"description":"Supplied tdm-policy URL."},"source":{"description":"header or meta"}}},"note":{"type":"string"},"witnesses":{"type":"array","items":{"type":"object","properties":{"ruleId":{"type":"string"},"detail":{"type":"string"},"path":{"type":"string","description":"JSON pointer or dotted path into the input."},"offset":{"type":"integer"},"index":{"type":"integer"},"expected":{"description":"What the rule required."},"actual":{"description":"What the input had."}}}},"versions":{"type":"object","description":"Every semantic dependency of this answer (engine, tables and their as-of dates, tokenizer, calendar, rounding profile)."},"limits":{"type":"array","items":{"type":"string"}},"deterministic":{"type":"boolean","description":"Always true: pure code, no model."}}},"_provenance":{"type":"object","description":"C2PA-aligned manifest and W3C VC 2.0 (eddsa-jcs-2022) issued by the hub DID; verify at POST /api/v1/verify-provenance"}},"required":["data","_provenance"]},"example":{"userAgent":"foobot","paths":["/example/page.html","/example/other.html","/index.html"],"robotsTxt":"User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\n\nUser-Agent: foobot\nDisallow:/\nAllow:/example/page.html\nAllow:/example/allowed.gif\n"},"openapiOperation":{"operationId":"robots_check","summary":"robots.txt (RFC 9309) rule reading with llms.txt and AI-use signals","description":"Reads a supplied robots.txt the RFC 9309 way for an agent deciding whether to fetch up to 25 paths as one product-token user agent: group selection (merged groups, \"*\" fallback), longest-match with the Allow-wins-tie rule, \"*\" and \"$\" patterns, percent-encoding normalization, fetch-status handling (4xx = no rules, 5xx = disallow all), the rule and line that decided each path. It also lints a supplied llms.txt (llmstxt.org format) and lists supplied X-Robots-Tag / meta robots / noai / TDM-reservation signals. This is a reading of published rules, not permission: it does NOT fetch the file, interpret terms of service or licences, follow redirects, honour Crawl-delay, or prove the file you supplied is the live one.\n\nPrice: 0.005 USDC on Base (x402).","x-x402-price-usdc":"0.005","x-x402-amount":"5000","x-payment-info":{"protocols":["x402"],"price":{"mode":"fixed","currency":"USD","amount":"0.005"}},"x-audience":"developer","x-sample-url":"https://agents.writerslogic.com/api/v1/samples/robots-check","security":[{"x402":[]},{"apiKey":[]}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":false,"required":["userAgent","paths"],"properties":{"userAgent":{"type":"string","maxLength":64,"pattern":"^[A-Za-z_-]+$","description":"Product token only (letters, underscore, hyphen), e.g. \"Googlebot\"; no version."},"paths":{"type":"array","minItems":1,"maxItems":25,"items":{"type":"string","maxLength":2048},"description":"Path plus optional query beginning with /, or a full http(s) URL (path and query are used)."},"robotsTxt":{"type":"string","maxLength":512000,"description":"robots.txt body, at most 512,000 UTF-8 bytes (500 KiB)."},"fetchStatus":{"type":"integer","minimum":100,"maximum":599,"description":"HTTP status the robots.txt fetch ended with (after at most 5 redirects). Omit when robotsTxt is supplied and was a 2xx."},"purpose":{"type":"string","enum":["crawl","ai-training"],"description":"ai-training also treats noai, noimageai and tdm-reservation: 1 as restrictions. Default crawl."},"llmsTxt":{"type":"string","maxLength":200000},"headers":{"type":"array","maxItems":50,"items":{"type":"object","required":["name","value"],"additionalProperties":false,"properties":{"name":{"type":"string","maxLength":100},"value":{"type":"string","maxLength":2000}}},"description":"Response headers of the page: X-Robots-Tag, tdm-reservation, tdm-policy."},"metaTags":{"type":"array","maxItems":50,"items":{"type":"object","required":["name","content"],"additionalProperties":false,"properties":{"name":{"type":"string","maxLength":100},"content":{"type":"string","maxLength":2000}}},"description":"HTML meta elements: robots, googlebot, tdm-reservation, tdm-policy."}}},"example":{"userAgent":"foobot","paths":["/example/page.html","/example/other.html","/index.html"],"robotsTxt":"User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\n\nUser-Agent: foobot\nDisallow:/\nAllow:/example/page.html\nAllow:/example/allowed.gif\n"}}}},"responses":{"200":{"description":"Paid result with provenance","content":{"application/json":{"schema":{"type":"object","properties":{"data":{"type":"object","properties":{"status":{"type":"string","enum":["pass","fail","unknown","unsupported"],"description":"pass: every rule held. fail: a rule was violated (see witnesses). unknown: the input lacks the evidence to decide. unsupported: outside what this engine covers. Missing evidence is never a pass."},"fileUsed":{"type":"string","enum":["parsed","unavailable-allow-all","unreachable-disallow-all","none"]},"matchedGroup":{"type":"string","enum":["agent","*","none"]},"decisions":{"type":"array","items":{"type":"object","properties":{"path":{"type":"string"},"allowed":{"type":"boolean"},"reason":{"type":"string"},"matched":{"description":"Deciding rule: directive, pattern, line, octets; null when none matched."},"tie":{"type":"boolean","description":"An equal-length rule of the other kind also matched."}}}},"sitemaps":{"type":"array","items":{"type":"string"}},"ignoredRulesBeforeGroup":{"type":"integer"},"llms":{"description":"llms.txt lint: title, summary, sections, link count and problems; null when not supplied."},"signals":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"name":{"type":"string"},"agent":{"description":"User-agent token the directive targets, or null for all."},"token":{"type":"string"},"effect":{"type":"string"}}}},"tdm":{"type":"object","properties":{"reservation":{"type":"string","enum":["reserved","not-reserved","unset"]},"raw":{"description":"Supplied tdm-reservation value."},"policy":{"description":"Supplied tdm-policy URL."},"source":{"description":"header or meta"}}},"note":{"type":"string"},"witnesses":{"type":"array","items":{"type":"object","properties":{"ruleId":{"type":"string"},"detail":{"type":"string"},"path":{"type":"string","description":"JSON pointer or dotted path into the input."},"offset":{"type":"integer"},"index":{"type":"integer"},"expected":{"description":"What the rule required."},"actual":{"description":"What the input had."}}}},"versions":{"type":"object","description":"Every semantic dependency of this answer (engine, tables and their as-of dates, tokenizer, calendar, rounding profile)."},"limits":{"type":"array","items":{"type":"string"}},"deterministic":{"type":"boolean","description":"Always true: pure code, no model."}}},"_provenance":{"type":"object","description":"C2PA-aligned manifest and W3C VC 2.0 (eddsa-jcs-2022) issued by the hub DID; verify at POST /api/v1/verify-provenance"}},"required":["data","_provenance"]}}}},"400":{"description":"Invalid request (no payment consumed)","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"},"code":{"type":"string"}}}}}},"401":{"description":"Invalid X-API-KEY","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"},"code":{"type":"string"}}}}}},"402":{"description":"Payment required or invalid","content":{"application/json":{"schema":{"type":"object","description":"x402 v2 PaymentRequired (also base64 in the PAYMENT-REQUIRED header)","properties":{"x402Version":{"type":"integer"},"error":{"type":"string"},"code":{"type":"string"},"resource":{"type":"object"},"accepts":{"type":"array","items":{"type":"object"}},"extensions":{"type":"object"}}}}}},"502":{"description":"Upstream failure; the payment is released or not settled","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"string"},"code":{"type":"string"}}}}}}}},"links":{"docs":"https://agents.writerslogic.com/docs/robots-check/","docsJson":"https://agents.writerslogic.com/docs/robots-check.json","sample":"https://agents.writerslogic.com/api/v1/samples/robots-check","openapi":"https://agents.writerslogic.com/openapi.json","catalog":"https://agents.writerslogic.com/api/v1/catalog","verifyProvenance":"https://agents.writerslogic.com/api/v1/verify-provenance","index":"https://agents.writerslogic.com/docs/index.json"}}