{"endpoint":{"id":"1a67c8fc21c7a14c","rail":"x402","source":"cdp-bazaar","url":"https://web.openverbs.com/v1/robots","method":"POST","host":"web.openverbs.com","serviceName":"Check robots.txt crawl rules","description":"Fetch a site's robots.txt and evaluate crawl permissions: given a URL (plus optional extra paths) and a user-agent, return whether each path is allowed or disallowed, the rule that matched, the user-agent group, the crawl-delay and any declared sitemaps. Implements the Robots Exclusion Protocol (RFC 9309) with longest-match-wins, Allow-over-Disallow tie-breaking and * / $ wildcards.","tags":["robots.txt parser","robots txt checker","is url crawlable","crawl permission api","user-agent allow disallow"],"categoryId":"web-extraction","priceUsd":0.006,"priceScheme":"exact","network":"eip155:8453","payTo":"0xb3f4de103F34b99f0Da19B4F16D6B53a14C2194A","testnetOnly":false,"calls30d":3,"payers30d":3,"lastCalledAt":"2026-10-08T05:38:34.94Z","firstSeenAt":"2026-10-10T17:11:08.641Z","lastSeenAt":"2026-10-11T12:08:31.983Z","lastProbedAt":"2026-10-11T08:30:40.094Z","embeddedHash":"8caf80aef987e2857aee7f6f994261bea8d37aace6be438d3c0d5ed906bfc2b5"},"score":{"endpointId":"1a67c8fc21c7a14c","computedAt":"2026-10-11T11:46:03.128Z","score":77,"grade":"B","uptime7d":1,"latencyP50Ms":270.5,"probeCount7d":4,"flags":[]},"recentProbes":[{"id":12342,"endpointId":"1a67c8fc21c7a14c","checkedAt":"2026-10-11T08:30:40.265Z","statusCode":402,"latencyMs":172,"valid402":true,"priceUsd":0.006,"payTo":"0xb3f4de103F34b99f0Da19B4F16D6B53a14C2194A","error":null},{"id":8804,"endpointId":"1a67c8fc21c7a14c","checkedAt":"2026-10-11T04:15:30.256Z","statusCode":402,"latencyMs":320,"valid402":true,"priceUsd":0.006,"payTo":"0xb3f4de103F34b99f0Da19B4F16D6B53a14C2194A","error":null},{"id":5264,"endpointId":"1a67c8fc21c7a14c","checkedAt":"2026-10-10T23:45:53.547Z","statusCode":402,"latencyMs":402,"valid402":true,"priceUsd":0.006,"payTo":"0xb3f4de103F34b99f0Da19B4F16D6B53a14C2194A","error":null},{"id":1045,"endpointId":"1a67c8fc21c7a14c","checkedAt":"2026-10-10T18:30:23.056Z","statusCode":402,"latencyMs":221,"valid402":true,"priceUsd":0.006,"payTo":"0xb3f4de103F34b99f0Da19B4F16D6B53a14C2194A","error":null}]}