{"endpoint":{"id":"c954e76abd2edf08","rail":"x402","source":"cdp-bazaar","url":"https://netintel.dev/robots-txt/analyze","method":"GET","host":"netintel.dev","serviceName":"NetIntel","description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","tags":["robots.txt","crawl rules","seo","web intelligence","scraping"],"categoryId":"web-extraction","priceUsd":0.01,"priceScheme":"exact","network":"eip155:8453","payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","testnetOnly":false,"calls30d":4,"payers30d":3,"lastCalledAt":"2026-10-08T06:08:15.375Z","firstSeenAt":"2026-10-10T17:10:46.533Z","lastSeenAt":"2026-10-11T12:08:11.063Z","lastProbedAt":"2026-10-11T12:46:20.389Z","embeddedHash":"7eebf6cb11ecf990f04b19294fcbee2745a626f3d27c9a69cf5c81309cc63138"},"score":{"endpointId":"c954e76abd2edf08","computedAt":"2026-10-11T12:46:12.076Z","score":77,"grade":"B","uptime7d":1,"latencyP50Ms":400,"probeCount7d":4,"flags":[]},"recentProbes":[{"id":15737,"endpointId":"c954e76abd2edf08","checkedAt":"2026-10-11T12:46:20.536Z","statusCode":402,"latencyMs":1330,"valid402":true,"priceUsd":0.01,"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","error":null},{"id":12097,"endpointId":"c954e76abd2edf08","checkedAt":"2026-10-11T08:15:48.291Z","statusCode":402,"latencyMs":267,"valid402":true,"priceUsd":0.01,"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","error":null},{"id":8536,"endpointId":"c954e76abd2edf08","checkedAt":"2026-10-11T03:45:58.865Z","statusCode":402,"latencyMs":533,"valid402":true,"priceUsd":0.01,"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","error":null},{"id":5056,"endpointId":"c954e76abd2edf08","checkedAt":"2026-10-10T23:30:35.394Z","statusCode":402,"latencyMs":757,"valid402":true,"priceUsd":0.01,"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","error":null},{"id":653,"endpointId":"c954e76abd2edf08","checkedAt":"2026-10-10T18:00:26.448Z","statusCode":402,"latencyMs":165,"valid402":true,"priceUsd":0.01,"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","error":null}]}