Cloudflare rolls out \u2018pay-per-crawl\u2019 feature to constrain AI\u2019s limitless hunger for data | CyberScoop<\/title> <meta name=\"description\" content=\"Cloudflare announced Tuesday it will allow customers to block or charge fees for web crawlers deployed to scrape their websites and data on behalf of AI systems.\"> <link rel=\"canonical\" href=\"https:\/\/cyberscoop.com\/cloudflare-ai-web-crawlers-pay-per-crawl-websites-data\/\"> <meta property=\"og:locale\" content=\"en_US\"> <meta property=\"og:type\" content=\"article\"> <meta property=\"og:title\" content=\"Cloudflare rolls out \u2018pay-per-crawl\u2019 feature to constrain AI\u2019s limitless hunger for data \"> <meta property=\"og:description\" content=\"Cloudflare announced Tuesday it will allow customers to block or charge fees for web crawlers deployed to scrape their websites and data on behalf of AI systems.\"> <meta property=\"og:url\" content=\"https:\/\/cyberscoop.com\/cloudflare-ai-web-crawlers-pay-per-crawl-websites-data\/\"> <meta property=\"og:site_name\" content=\"CyberScoop\"> <meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/cyberscoop\/\"> <meta property=\"article:published_time\" content=\"2025-07-01T19:43:31+00:00\"> <meta property=\"article:modified_time\" content=\"2025-07-01T19:43:34+00:00\"> <meta property=\"og:image\" content=\"https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg\"> <meta property=\"og:image:width\" content=\"5800\"> <meta property=\"og:image:height\" content=\"3258\"> <meta property=\"og:image:type\" content=\"image\/jpeg\"> <meta name=\"author\" content=\"djohnson\"> <meta name=\"twitter:card\" content=\"summary_large_image\"> <meta name=\"twitter:creator\" content=\"@CyberScoopNews\"> <meta name=\"twitter:site\" content=\"@CyberScoopNews\">  <link rel=\"dns-prefetch\" href=\"\/\/securepubads.g.doubleclick.net\">\n<link rel=\"dns-prefetch\" href=\"\/\/use.typekit.net\">\n<link rel=\"alternate\" type=\"application\/rss+xml\" title=\"CyberScoop \u00bb Feed\" href=\"https:\/\/cyberscoop.com\/feed\/\">\n<link rel=\"alternate\" type=\"application\/rss+xml\" title=\"CyberScoop \u00bb Comments Feed\" href=\"https:\/\/cyberscoop.com\/comments\/feed\/\"> <link rel=\"stylesheet\" id=\"all-css-2\" href=\"https:\/\/cyberscoop.com\/wp-includes\/css\/dist\/block-library\/style.min.css?m=1746040294g\" type=\"text\/css\" media=\"all\"> <link rel=\"stylesheet\" id=\"all-css-6\" href=\"https:\/\/cyberscoop.com\/wp-content\/mu-plugins\/search\/elasticpress\/dist\/css\/related-posts-block-styles.min.css?m=1750115417g\" type=\"text\/css\" media=\"all\"> <link rel=\"stylesheet\" id=\"all-css-8\" href=\"https:\/\/cyberscoop.com\/wp-content\/themes\/scoopnewsgroup\/dist\/css\/frontend.css?m=1748220166g\" type=\"text\/css\" media=\"all\">\n<link rel=\"stylesheet\" id=\"typekit-css\" href=\"https:\/\/use.typekit.net\/itk2qbh.css?ver=13897d660a0ac2c9c7d1\" media=\"all\"> <link rel=\"https:\/\/api.w.org\/\" href=\"https:\/\/cyberscoop.com\/wp-json\/\"><link rel=\"alternate\" title=\"JSON\" type=\"application\/json\" href=\"https:\/\/cyberscoop.com\/wp-json\/wp\/v2\/posts\/85051\"><link rel=\"EditURI\" type=\"application\/rsd+xml\" title=\"RSD\" href=\"https:\/\/cyberscoop.com\/xmlrpc.php?rsd\">\n<meta name=\"generator\" content=\"WordPress 6.8.1\">\n<link rel=\"shortlink\" href=\"https:\/\/cyberscoop.com\/?p=85051\">\n<link rel=\"alternate\" title=\"oEmbed (JSON)\" type=\"application\/json+oembed\" href=\"https:\/\/cyberscoop.com\/wp-json\/oembed\/1.0\/embed?url=https%3A%2F%2Fcyberscoop.com%2Fcloudflare-ai-web-crawlers-pay-per-crawl-websites-data%2F\">\n<link rel=\"alternate\" title=\"oEmbed (XML)\" type=\"text\/xml+oembed\" href=\"https:\/\/cyberscoop.com\/wp-json\/oembed\/1.0\/embed?url=https%3A%2F%2Fcyberscoop.com%2Fcloudflare-ai-web-crawlers-pay-per-crawl-websites-data%2F&format=xml\">   <link rel=\"icon\" href=\"https:\/\/cyberscoop.com\/wp-content\/uploads\/sites\/3\/2023\/01\/cropped-cs_favicon-2.png?w=32\" sizes=\"32x32\">\n<link rel=\"icon\" href=\"https:\/\/cyberscoop.com\/wp-content\/uploads\/sites\/3\/2023\/01\/cropped-cs_favicon-2.png?w=192\" sizes=\"192x192\">\n<link rel=\"apple-touch-icon\" href=\"https:\/\/cyberscoop.com\/wp-content\/uploads\/sites\/3\/2023\/01\/cropped-cs_favicon-2.png?w=180\">\n<meta name=\"msapplication-TileImage\" content=\"https:\/\/cyberscoop.com\/wp-content\/uploads\/sites\/3\/2023\/01\/cropped-cs_favicon-2.png?w=270\"> <\/head><body class=\"wp-singular post-template-default single single-post postid-85051 single-format-standard wp-theme-scoopnewsgroup wp-child-theme-cyberscoop\" id=\"readabilityBody\"> <a href=\"https:\/\/cyberscoop.com\/cloudflare-ai-web-crawlers-pay-per-crawl-websites-data\/#main\" class=\"skip-to-content-link visually-hidden-focusable\">Skip to main content<\/a> <\/p>\n<div class=\"ad ad--top ad--top-desktop\">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p> <main id=\"main\" role=\"main\" tabindex=\"-1\"> <\/p>\n<div class=\"ad ad--top ad--top-mobile\">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<section id=\"stickybar\" class=\"stickybar stickybar--event js-stickybar\">\n<div class=\"stickybar__details\" readability=\"4.9180327868852\">\n<div class=\"stickybar__info js-sticky-bar-content\" readability=\"32\">\n<p>Voting is open for the 2025 CyberScoop 50 awards!<\/p>\n<\/div>\n<p> <a class=\"stickybar__link button button-tertiary\" href=\"https:\/\/cyberscoop.com\/cyberscoop50\/\">Click here!<\/a> <\/div>\n<p> <button class=\"stickybar__close js-stickybar-close\" aria-controls=\"stickybar\"> <svg class=\"icon icon--close\" width=\"21\" height=\"22\" viewBox=\"0 0 21 22\" fill=\"none\"><path d=\"m.822.518-.805.805L9.695 11 .017 20.678l.805.805 9.678-9.678 9.677 9.678.806-.805L11.305 11l9.678-9.677-.806-.805-9.677 9.677L.822.518Z\" fill=\"currentColor\" \/><\/svg> <span class=\"visually-hidden\">Close<\/span> <\/button> <\/section>\n<article class=\"single-article content\">\n<div class=\"single-article__container js-single-article-content\">\n<header class=\"single-article__header \" readability=\"26.041420118343\">\n<div class=\"single-article__header-content\" readability=\"35.52\">\n<p> The move is the result customer feedback, since they neither wanted to grant AI web crawlers unrestricted access to their data nor block the practice entirely. <\/p>\n<p>  <br \/>\n<audio id=\"audio-player\" src=\"https:\/\/wp-tts-cdn.api.scpnewsgrp.com\/cyberscoop\/85051\/english.openai.mp3\"><\/audio> <\/p>\n<div readability=\"11\">\n<div>\n<p>Listen to this article<\/p>\n<p>  <\/p>\n<p>0:00<\/p>\n<\/p><\/div>\n<p>  <\/p>\n<p> <span id=\"tts-tooltip\">Learn more.<\/span> <span> This feature uses an automated voice, which may result in occasional errors in pronunciation, tone, or sentiment. <\/span> <\/p>\n<\/div>\n<p>  <\/div>\n<div class=\"single-article__cover-wrap\">\n<figure class=\"single-article__cover\"> <img data-recalc-dims=\"1\" fetchpriority=\"high\" width=\"640\" height=\"360\" src=\"https:\/\/i0.wp.com\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data.jpg?resize=640%2C360&ssl=1\" class=\"single-article__cover-image wp-post-image\" alt decoding=\"async\" fetchpriority=\"high\" srcset=\"https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg 5800w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=300,168 300w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=768,431 768w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=1024,575 1024w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=1536,863 1536w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=2048,1150 2048w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=600,337 600w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=1200,675 1200w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-2.jpg?resize=1500,843 1500w\" sizes=\"(max-width: 1200px) 100vw, 1200px\"><figcaption> Cloudflare headquarters in San Francisco. (Getty Images) <\/figcaption><\/figure>\n<\/p><\/div>\n<\/header>\n<div class=\"single-article__content\">\n<div class=\"single-article__content-inner has-drop-cap\"> <html readability=\"52.611432985709\"><body readability=\"105.57761578045\"><\/p>\n<p>Cloudflare announced Tuesday it will allow customers to block or charge fees for web crawlers deployed to scrape their websites and data on behalf of AI systems.<\/p>\n<p>In a blog on its corporate website, Will Allen, Cloudflare\u2019s vice president of product, and Simon Newton, an engineer manager, said the company is establishing a new system to limit AI web crawlers after hearing feedback from their customers.<\/p>\n<p>The beta feature, called \u201cPay-Per-Crawl,\u201d integrates with web infrastructure, HTTP status codes and authentication mechanisms that allow for paid content access to customer websites.<\/p>\n<p>The move is the result of feedback that the company \u2014 which provides hosting and cybersecurity services for<a href=\"https:\/\/blog.cloudflare.com\/application-security\/\"> approximately 1 out of every 5<\/a> websites in the world \u2014 got from customers, who neither wanted to grant AI web crawlers unrestricted access to their data nor block the practice entirely. <\/p>\n<div class=\"ad ad--inline_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p>\u201cAfter hundreds of conversations with news organizations, publishers, and large-scale social media platforms, we heard a consistent desire for a third path: They\u2019d like to allow AI crawlers to access their content, but they\u2019d like to get compensated,\u201d Allen and Newton wrote.<\/p>\n<p>Domain owners can designate a flat fee for every request, giving publishers the option to block the crawler entirely, allow free access or pay a domain-wide price for access. Cloudflare will act as both the Merchant of Record for the exchanges and provide the underlying technical infrastructure to run pay-per-crawl.<\/p>\n<p>AI crawlers, meanwhile, will be able to register with the system, see pricing options for different resources and set maximum price points for the system to decide whether it\u2019s worth the costs, according to a Cloudflare\u2019s sign-up page for the beta program. To prevent bad actors from spoofing legitimate crawlers and collecting fraudulent payments, AI crawlers must also register with Cloudflare and provide the URL of their key directories and user agent information.<\/p>\n<p>An<a href=\"https:\/\/www.cloudflare.com\/paypercrawl-signup\/\"> entry<\/a> in the frequently asked questions section says the company \u2014 which handles trillions of requests and beats back automated denial-of-service attacks every day \u2014 has \u201cthe world\u2019s most advanced bot management solutions,\u201d using a combination of machine learning, behavioral analysis and digital fingerprinting to separate AI crawlers from search engine bots, verified bot programs and other \u201cgood\u201d forms of automated web scraping.<\/p>\n<p>The announcement marks a potentially substantial blow to one of the primary ways that AI models feed and train their systems: by collecting every scrap of publicly available data they can through web-scraping technology.<\/p>\n<div class=\"ad ad--inline_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p>Web scraping is far from new, but the data-hungry needs of large language models<a href=\"https:\/\/www.eff.org\/deeplinks\/2025\/06\/keeping-web-under-weight-ai-crawlers\"> has grown the practice<\/a> to potentially unsustainable levels, eating up traffic bandwidth, causing sites to load more slowly and other service disruptions. The Wikimedia Foundation said that since January 2024, 65% of its most expensive traffic has come from bots. It has also seen bandwidth used for downloading multimedia content grow by 50%, noting that the expansion of AI scrapers \u201cis causing a significant load on the underlying infrastructure that keep our sites available for everyone.\u201d<\/p>\n<p>\u201cWe are observing a significant increase in request volume, with most of this traffic being driven by scraping bots collecting training data for large language models (LLMs) and other use cases,\u201d members of foundation<a href=\"https:\/\/diff.wikimedia.org\/2025\/04\/01\/how-crawlers-impact-the-operations-of-the-wikimedia-projects\/\"> wrote<\/a> in April. \u201cAutomated requests for our content have grown exponentially, alongside the broader technology economy, via mechanisms including scraping, APIs, and bulk downloads.\u201d<\/p>\n<p>Allen and Newton wrote that features like pay-per-crawl are part of a larger shift in \u201chow content is controlled online\u201d as AI systems gobble up data from every available source. They expect the program to evolve \u201csignificantly\u201d over the years to cover different types of transactions and marketplaces.<\/p>\n<p>\u201cFor example, a publisher or new organization might want to charge different rates for different paths or content types,\u201d the authors wrote. \u201cHow do you introduce dynamic pricing based not only upon demand, but also how many users your AI application has? How do you introduce granular licenses at internet scale, whether for training, inference, search, or something entirely new?\u201d<\/p>\n<p>The move to establish a system of compensation for websites and data owners comes as AI companies like OpenAI are facing numerous copyright lawsuits from artists, writers, publishers and other content creators who argue AI companies are training their systems and profiting from content produced by others.<\/p>\n<p><\/body> <\/p>\n<footer class=\"single-article__footer\" readability=\"3.812865497076\">\n<div class=\"author-card\" readability=\"13\">\n<div class=\"author-card__avatar\">\n<figure class=\"author-card__image-wrap\"> <img data-recalc-dims=\"1\" decoding=\"async\" class=\"author-card__image\" src=\"https:\/\/i0.wp.com\/ddi.mohflo.net\/wp-content\/uploads\/2025\/07\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data-1.jpg?w=640&ssl=1\" alt=\"Derek B. Johnson\"> <\/figure>\n<\/p><\/div>\n<p><h4 class=\"author-card__name\">Written by Derek B. Johnson<\/h4>\n<p> Derek B. Johnson is a reporter at CyberScoop, where his beat includes cybersecurity, elections and the federal government. Prior to that, he has provided award-winning coverage of cybersecurity news across the public and private sectors for various publications since 2017. Derek has a bachelor\u2019s degree in print journalism from Hofstra University in New York and a master\u2019s degree in public policy from George Mason University in Virginia. <\/p>\n<\/p><\/div>\n<div class=\"single-article__tags-container\">\n<h4 class=\"single-article__tags-title\">In This Story<\/h4>\n<\/p><\/div>\n<\/footer>\n<p> <\/html><\/div>\n<\/p><\/div>\n<\/p><\/div>\n<div class=\"single-article__ads js-single-article-sidebar\">\n<div class=\"ad ad--sidebar js-single-article-sidebar-5 ad--rightrail_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<div class=\"ad ad--sidebar js-single-article-sidebar-4 ad--rightrail_2 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<div class=\"ad ad--sidebar js-single-article-sidebar-3 ad--rightrail_3 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div><\/div>\n<\/article>\n<section class=\"latest-podcasts\">\n<h2 class=\"latest-podcasts__title\"> Latest Podcasts\t<\/h2>\n<\/section>\n<div class=\"top-categories\">\n<div class=\"top-categories__container\">\n<h3 class=\"top-categories__category-title\">Government<\/h3>\n<\/p><\/div>\n<div class=\"top-categories__container\">\n<h3 class=\"top-categories__category-title\">Technology<\/h3>\n<\/p><\/div>\n<\/p><\/div>\n<p> <\/main> <\/p>\n<div class=\"ad ad--bottom \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<div id=\"interstitial\" class=\"welcome__container\"> <button id=\"close-modal-1\" class=\"welcome__clickable_area\"><\/button> <\/p>\n<div class=\"welcome__ad_wrapper\">\n<p> <button id=\"close-modal-3\" class=\"welcome__continue-button\">Continue to CyberScoop<\/button> <\/p>\n<\/p><\/div>\n<\/p><\/div>\n<p>   <\/body> <a href=\"https:\/\/cyberscoop.com\/cloudflare-ai-web-crawlers-pay-per-crawl-websites-data\/\">Source<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>Cloudflare rolls out \u2018pay-per-crawl\u2019 feature to constrain AI\u2019s limitless hunger<\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[235,632,78,310,4519],"tags":[236,633,86,311,4520],"class_list":["post-7774","post","type-post","status-publish","format-standard","hentry","category-ai","category-cloudflare","category-cybersecurity","category-technology","category-web-scraping","tag-ai","tag-cloudflare","tag-cybersecurity","tag-technology","tag-web-scraping"],"featured_image_urls":{"full":"","thumbnail":"","medium":"","medium_large":"","large":"","1536x1536":"","2048x2048":"","chromenews-featured":"","chromenews-large":"","chromenews-medium":""},"author_info":{"display_name":"Cyber Scoop","author_link":"https:\/\/ddi.mohflo.net\/index.php\/author\/cyberscoop\/"},"category_info":"<a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/ai\/\" rel=\"category tag\">AI<\/a> <a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/cloudflare\/\" rel=\"category tag\">Cloudflare<\/a> <a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/cybersecurity\/\" rel=\"category tag\">Cybersecurity<\/a> <a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/technology\/\" rel=\"category tag\">Technology<\/a> <a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/web-scraping\/\" rel=\"category tag\">web scraping<\/a>","tag_info":"web scraping","comment_count":"0","jetpack_featured_media_url":"","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/posts\/7774","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/comments?post=7774"}],"version-history":[{"count":0,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/posts\/7774\/revisions"}],"wp:attachment":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/media?parent=7774"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/categories?post=7774"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/tags?post=7774"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}

{"id":7774,"date":"2025-07-01T14:43:31","date_gmt":"2025-07-01T19:43:31","guid":{"rendered":"https:\/\/cyberscoop.com\/?p=85051"},"modified":"2025-07-01T14:43:31","modified_gmt":"2025-07-01T19:43:31","slug":"cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data","status":"publish","type":"post","link":"https:\/\/ddi.mohflo.net\/index.php\/2025\/07\/01\/cloudflare-rolls-out-pay-per-crawl-feature-to-constrain-ais-limitless-hunger-for-data\/","title":{"rendered":"Cloudflare rolls out \u2018pay-per-crawl\u2019 feature to constrain AI\u2019s limitless hunger for data\u00a0"},"content":{"rendered":"