{"endpoint":{"id":"54bc8e06fa6b6214","rail":"x402","source":"cdp-bazaar","url":"https://bess-x402.onrender.com/v1/web/robots","method":"GET","host":"bess-x402.onrender.com","serviceName":"Robots.txt & Sitemap","description":"A site's robots.txt rules per user agent, whether AI crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot, PerplexityBot) are blocked, and the URLs listed in its sitemap. Use before crawling a site or for SEO audits.","tags":["seo","robots","sitemap","crawling","ai-crawlers"],"platforms":[],"tasks":["crawl","robots"],"categoryId":"web-extraction","priceUsd":0.002,"priceScheme":"exact","network":"eip155:8453","payTo":"0x1129A6BB9C6165FF07fdC509c107B522CA0e1871","testnetOnly":false,"calls30d":null,"payers30d":null,"lastCalledAt":"2026-10-22T06:53:56.519Z","startFeeUsd":null,"platformUsageExtra":null,"reportedSuccess30d":null,"pageUrl":null,"batchMax":null,"exampleCount":20,"unitsPerCharge":1,"unitBasis":"call","unitPriceUsd":0.002,"firstSeenAt":"2026-10-22T12:08:01.884Z","lastSeenAt":"2026-10-22T12:08:01.884Z","lastProbedAt":"2026-10-22T12:15:08.496Z","embeddedHash":"bed28cbe2de8674f48eee9b9ee11c625e6564391b7f4e415a74f8c57b7f88423"},"score":{"endpointId":"54bc8e06fa6b6214","computedAt":"2026-10-22T14:46:25.347Z","score":null,"grade":null,"uptime7d":1,"latencyP50Ms":193,"probeCount7d":1,"flags":["no-usage-data"]},"recentProbes":[{"id":288430,"endpointId":"54bc8e06fa6b6214","checkedAt":"2026-10-22T12:15:08.638Z","statusCode":402,"latencyMs":193,"valid402":true,"priceUsd":0.002,"payTo":"0x1129A6BB9C6165FF07fdC509c107B522CA0e1871","error":null,"mcpOutcome":null,"mcpProtocol":null}],"siblings":{"total":30,"sameCategory":2,"rows":[{"id":"e942c3b214fa5915","method":"GET","url":"https://bess-x402.onrender.com/v1/apps/search","categoryId":"web-search","priceUsd":0.005,"priceScheme":"exact"},{"id":"32eff3eac7f7a602","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/balance","categoryId":"onchain-data","priceUsd":0.003,"priceScheme":"exact"},{"id":"9932b8ad78095b79","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/btc-fees","categoryId":"onchain-data","priceUsd":0.002,"priceScheme":"exact"},{"id":"b2c05ed55e946025","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/candles","categoryId":"crypto-prices","priceUsd":0.002,"priceScheme":"exact"},{"id":"52b69407e77fb49c","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/coin","categoryId":"crypto-prices","priceUsd":0.003,"priceScheme":"exact"},{"id":"b4220da3e6ce7805","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/dex-pairs","categoryId":"crypto-prices","priceUsd":0.003,"priceScheme":"exact"},{"id":"d2a1b2fd7d2772d6","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/news","categoryId":"news","priceUsd":0.003,"priceScheme":"exact"},{"id":"25c1f62d6e5cb758","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/price","categoryId":"crypto-prices","priceUsd":0.002,"priceScheme":"exact"},{"id":"dfe88ccb67b71590","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/price-history","categoryId":"crypto-prices","priceUsd":0.003,"priceScheme":"exact"},{"id":"5eda5f03ac451104","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/protocols","categoryId":"crypto-prices","priceUsd":0.003,"priceScheme":"exact"},{"id":"7f6ce8379806c702","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/solana-balance","categoryId":"onchain-data","priceUsd":0.003,"priceScheme":"exact"},{"id":"174f0d5a0830e40d","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/ticker","categoryId":"crypto-prices","priceUsd":0.002,"priceScheme":"exact"},{"id":"6a1c697eca80ff41","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/top-coins","categoryId":"crypto-prices","priceUsd":0.002,"priceScheme":"exact"},{"id":"7d06fe300afc8f89","method":"GET","url":"https://bess-x402.onrender.com/v1/crypto/trending","categoryId":"crypto-prices","priceUsd":0.002,"priceScheme":"exact"},{"id":"149cceaf2d713898","method":"GET","url":"https://bess-x402.onrender.com/v1/finance/fx","categoryId":"stocks","priceUsd":0.002,"priceScheme":"exact"},{"id":"608559eb98e90d90","method":"GET","url":"https://bess-x402.onrender.com/v1/geo/weather","categoryId":"weather","priceUsd":0.002,"priceScheme":"exact"},{"id":"8648937b413f9524","method":"GET","url":"https://bess-x402.onrender.com/v1/lookup/food","categoryId":"ecommerce","priceUsd":0.002,"priceScheme":"exact"},{"id":"f4ffab25d6f440f7","method":"GET","url":"https://bess-x402.onrender.com/v1/news/hackernews","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"de4a959d3dcb9a23","method":"GET","url":"https://bess-x402.onrender.com/v1/predictions/kalshi/search","categoryId":"crypto-prices","priceUsd":0.003,"priceScheme":"exact"},{"id":"1376ed993f4cab73","method":"GET","url":"https://bess-x402.onrender.com/v1/text/language","categoryId":"translation","priceUsd":0.002,"priceScheme":"exact"},{"id":"dae8c8da53dc0325","method":"GET","url":"https://bess-x402.onrender.com/v1/us-stocks/filings","categoryId":"stocks","priceUsd":0.003,"priceScheme":"exact"},{"id":"812b852690fec482","method":"GET","url":"https://bess-x402.onrender.com/v1/us-stocks/full-text-search","categoryId":"stocks","priceUsd":0.004,"priceScheme":"exact"},{"id":"2ffc1bd505cd6f12","method":"GET","url":"https://bess-x402.onrender.com/v1/us-stocks/fundamentals","categoryId":"stocks","priceUsd":0.005,"priceScheme":"exact"},{"id":"180ca6d13fb3402b","method":"GET","url":"https://bess-x402.onrender.com/v1/us-stocks/insiders","categoryId":"stocks","priceUsd":0.005,"priceScheme":"exact"},{"id":"5494efd4fe6807da","method":"GET","url":"https://bess-x402.onrender.com/v1/us-stocks/profile","categoryId":"stocks","priceUsd":0.003,"priceScheme":"exact"},{"id":"2889eb96acb45dcf","method":"GET","url":"https://bess-x402.onrender.com/v1/us-stocks/search","categoryId":"enrichment","priceUsd":0.002,"priceScheme":"exact"},{"id":"928c0d75d8ef86ad","method":"GET","url":"https://bess-x402.onrender.com/v1/util/holidays","categoryId":"stocks","priceUsd":0.002,"priceScheme":"exact"},{"id":"97119ee2b8144ce8","method":"GET","url":"https://bess-x402.onrender.com/v1/web/contacts","categoryId":"enrichment","priceUsd":0.005,"priceScheme":"exact"},{"id":"9ddb2989b466f2b6","method":"GET","url":"https://bess-x402.onrender.com/v1/web/links","categoryId":"web-extraction","priceUsd":0.002,"priceScheme":"exact"},{"id":"efdfc97b554107f3","method":"GET","url":"https://bess-x402.onrender.com/v1/web/markdown","categoryId":"web-extraction","priceUsd":0.005,"priceScheme":"exact"}]},"tools":null,"source":{"name":"Hagglyai","url":"https://hagglyai.com/compare/web-extraction","asOf":"2026-10-22T15:07:38.262Z"}}