{"endpoint":{"id":"5f42b734c3fc9870","rail":"x402","source":"cdp-bazaar","url":"https://api.webintel-api.workers.dev/api/security/robots-security-txt","method":"GET","host":"api.webintel-api.workers.dev","serviceName":"SiteCheck Web Intel","description":"Reads a site's public robots.txt (crawl rules, sitemaps, AI-crawler rules) and security.txt (RFC 9116: contact, expiry, policy), and says whether security.txt is present, current or expired. Missing files are reported as absent, not as errors.","tags":["security","robots","txt"],"platforms":[],"tasks":["crawl","robots"],"categoryId":"web-extraction","priceUsd":0.002,"priceScheme":"exact","network":"eip155:8453","payTo":"0xb8491F953e6860D3b5840b73c60883Ff099ceAfA","testnetOnly":false,"calls30d":null,"payers30d":null,"lastCalledAt":"2026-10-23T18:42:02.992Z","startFeeUsd":null,"platformUsageExtra":null,"reportedSuccess30d":null,"pageUrl":null,"batchMax":null,"exampleCount":null,"unitsPerCharge":1,"unitBasis":"call","unitPriceUsd":0.002,"firstSeenAt":"2026-10-24T00:08:27.106Z","lastSeenAt":"2026-10-24T00:08:27.106Z","lastProbedAt":"2026-10-24T00:30:09.550Z","embeddedHash":"a3747244bf256519ac3b23395fe3432278579ae053ef82ebd766e9f971f93806"},"score":{"endpointId":"5f42b734c3fc9870","computedAt":"2026-10-24T02:46:01.317Z","score":null,"grade":null,"uptime7d":1,"latencyP50Ms":535,"probeCount7d":1,"flags":["no-usage-data"]},"recentProbes":[{"id":326336,"endpointId":"5f42b734c3fc9870","checkedAt":"2026-10-24T00:30:09.679Z","statusCode":402,"latencyMs":535,"valid402":true,"priceUsd":0.002,"payTo":"0xb8491F953e6860D3b5840b73c60883Ff099ceAfA","error":null,"mcpOutcome":null,"mcpProtocol":null}],"siblings":{"total":25,"sameCategory":1,"rows":[{"id":"a791017ceed25e56","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/ap-news-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"8a921178394d5eb9","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/article-extractor","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"47e1c4e47ecbe373","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/bbc-news-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"90ca735807e8736e","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/benzinga","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"c20552283a1b9fc3","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/bing-search","categoryId":"web-search","priceUsd":0.002,"priceScheme":"exact"},{"id":"11b719066df797bf","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/bleacher-report-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"86cff03ecbff21bd","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/cnbc","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"57c8d0e87f485d0e","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/dappradar-rankings","categoryId":"crypto-prices","priceUsd":0.002,"priceScheme":"exact"},{"id":"ba700d0a9ffd9233","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/drive-com-au-news","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"8ac2587f0cffbeed","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/freightwaves","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"fcd57f4e2c9bdb26","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/ft-com-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"9b48ef56f28fd1ea","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/gdelt-news","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"5a49a5e21489e87a","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/google-news","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"abbb1f0788a9e7ad","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/google-search","categoryId":"web-search","priceUsd":0.002,"priceScheme":"exact"},{"id":"895dfcd28f63c0c5","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/markdown-to-pdf-mcp","categoryId":"pdf-generation","priceUsd":0.002,"priceScheme":"exact"},{"id":"322ef13ff86dabaf","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/news-across-the-web","categoryId":"news","priceUsd":0.003,"priceScheme":"exact"},{"id":"117da2bf12d13332","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/nyt-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"adb06c833a910451","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/pr-newswire","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"2d969faf7bf6b93e","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/press-release-multi-wire","categoryId":"news","priceUsd":0.003,"priceScheme":"exact"},{"id":"249857b77020463e","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/reuters-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"0c97f2367a510f47","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/shopify-metadata","categoryId":"ecommerce","priceUsd":0.002,"priceScheme":"exact"},{"id":"5301512dd8956d8b","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/techcrunch-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"59036d57dde9e534","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/usda-market-news","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"},{"id":"c16c04624d9f3641","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/website-content-crawler","categoryId":"web-extraction","priceUsd":0.002,"priceScheme":"exact"},{"id":"5e378ced249675fb","method":"GET","url":"https://api.webintel-api.workers.dev/api/search/wsj-articles","categoryId":"news","priceUsd":0.002,"priceScheme":"exact"}]},"tools":null,"source":{"name":"Hagglyai","url":"https://hagglyai.com/compare/web-extraction","asOf":"2026-10-24T03:19:53.246Z"}}