AI Crawlers are Driving Up Site Costs

{
    "jsonapi": {
        "version": "1.0",
        "meta": {
            "links": {
                "self": {
                    "href": "http:\/\/jsonapi.org\/format\/1.0\/"
                }
            }
        }
    },
    "data": {
        "type": "node--article",
        "id": "4e0ecfed-dacb-4360-8e7d-8e9b11149989",
        "links": {
            "self": {
                "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989?resourceVersion=id%3A568"
            }
        },
        "attributes": {
            "internalId": 566,
            "isPublished": true,
            "title": "AI Crawlers are Driving Up Site Costs",
            "created": "2024-05-03T15:49:17+00:00",
            "changed": "2024-06-10T19:18:41+00:00",
            "promote": false,
            "sticky": false,
            "metatag": [
                {
                    "tag": "meta",
                    "attributes": {
                        "name": "title",
                        "content": "AI Crawlers are Driving Up Site Costs | Inclind"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "name": "description",
                        "content": "In today\u0027s digital-first world, AI-powered web crawlers are increasingly traversing sites across the internet. While these bots play a crucial role in tasks"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "name": "abstract",
                        "content": "Delaware Drupal web development company specializing in custom Drupal development, real estate and rental brokerage websites, internet marketing and ecommerce"
                    }
                },
                {
                    "tag": "link",
                    "attributes": {
                        "rel": "canonical",
                        "href": "https:\/\/www.inclind.com\/news\/ai-crawlers-are-driving-site-costs"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "name": "robots",
                        "content": "index, follow"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:site_name",
                        "content": "Inclind"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:url",
                        "content": "https:\/\/www.inclind.com\/news\/ai-crawlers-are-driving-site-costs"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:title",
                        "content": "AI Crawlers are Driving Up Site Costs"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:description",
                        "content": "Delaware Drupal web development company specializing in custom Drupal development, real estate and rental brokerage websites, internet marketing and ecommerce"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "place:location:latitude",
                        "content": "38.77513"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "place:location:longitude",
                        "content": "-75.13974"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:street_address",
                        "content": "119 W. Third St., Suite 6"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:locality",
                        "content": "Lewes"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:region",
                        "content": "Delaware"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:postal_code",
                        "content": "19958"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:country_name",
                        "content": "United States"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:email",
                        "content": "hello@inclind.com"
                    }
                },
                {
                    "tag": "meta",
                    "attributes": {
                        "property": "og:phone_number",
                        "content": "(800) 604-8139"
                    }
                }
            ],
            "path": {
                "alias": "\/news\/ai-crawlers-are-driving-site-costs",
                "pid": 1603,
                "langcode": "en"
            },
            "body": {
                "value": "\u003Cp\u003EIn today\u0027s digital-first world, AI-powered web crawlers are increasingly traversing sites across the internet. While these bots play a crucial role in tasks like indexing for search engines and automating data collection, they also significantly increase server resource consumption. This surge can lead to higher operational costs for website owners, particularly in web hosting.\u003C\/p\u003E\u003Cdrupal-media data-entity-type=\u0022media\u0022 data-entity-uuid=\u002289f3f147-ce61-45c5-af41-22b695173dfa\u0022\u003E\u0026nbsp;\u003C\/drupal-media\u003E\u003Ch2\u003EThe Impact of AI Crawlers\u003C\/h2\u003E\u003Cp\u003EAI crawlers scan websites systematically, consuming bandwidth and processing power. This heightened activity not only strains servers but can also slow down the user experience for human visitors. For sites with limited hosting plans, this can translate into overage fees or upgrading to more expensive hosting options to accommodate the increased load.\u003C\/p\u003E\u003Cp\u003EFortunately, there are effective strategies to mitigate these impacts without compromising your site\u0027s visibility or functionality:\u003C\/p\u003E\u003Ch2\u003EMitigating Unwanted Bot Traffic\u003C\/h2\u003E\u003Col\u003E\u003Cli\u003E\u003Cstrong\u003EEnhancing Caching Configurations:\u003C\/strong\u003E\u003Cbr\u003EFor websites running on popular CMS platforms like \u003Ca href=\u0022\/services\/wordpress-websites\u0022\u003EWordPress\u003C\/a\u003E or \u003Ca href=\u0022\/services\/drupal-websites\u0022\u003EDrupal\u003C\/a\u003E, optimizing the caching configuration can significantly reduce server load. Effective caching ensures that static parts of your website are served to visitors without repeatedly processing the same requests. This is particularly useful during high-traffic periods induced by bot activity.\u003C\/li\u003E\u003Cli\u003E\u003Cstrong\u003EWAF (Web Application Firewall):\u003C\/strong\u003E\u003Cbr\u003ECloudflare or Fastly provides WAF services, offering a solution that allows website owners to block or rate limit bots based on their category. This selective filtering ensures that only beneficial bots can access your site, reducing unnecessary load.\u003C\/li\u003E\u003Cli\u003E\u003Cstrong\u003ECloudflare\u0027s Super Bot Fight Mode:\u003C\/strong\u003E\u003Cbr\u003ECloudflare\u0027s Super Bot Fight Mode offers robust protection against malicious bots by differentiating between harmful and beneficial bot traffic on your website. It allows site owners to block, challenge, or manage bots based on their behavior, providing a dashboard for detailed insights into bot activity. This feature is ideal for sites experiencing high levels of unwanted bot traffic, efficiently enhancing site performance and security.\u003C\/li\u003E\u003C\/ol\u003E\u003Cdrupal-media data-entity-type=\u0022media\u0022 data-entity-uuid=\u0022302683f0-b74a-4316-b00b-f8a9d79acf09\u0022\u003E\u0026nbsp;\u003C\/drupal-media\u003E\u003Ch2\u003EHow does this affect my Pantheon-hosted website?\u0026nbsp;\u003C\/h2\u003E\u003Cp\u003E\u003Ca href=\u0022https:\/\/pantheon.io\/\u0022 target=\u0022_blank\u0022\u003EPantheon\u003C\/a\u003E scales its web hosting based on visits. As of early 2024, it has \u003Ca href=\u0022https:\/\/pantheon.io\/blog\/enhanced-pantheon-overage-policy-traffic-limits-and-pricing\u0022 target=\u0022_blank\u0022\u003Eupdated its overage policy\u003C\/a\u003E to improve past scenarios where hosting plans were automatically upgraded if an overage occurred. It now charges $40 for each additional 10,000 site visitors over your site threshold.\u0026nbsp;\u003C\/p\u003E\u003Cp\u003EPantheon does offer preferred pricing and overage protections for \u003Ca href=\u0022\/clients\/non-profit-organizations\u0022\u003E501(c)3 nonprofits\u003C\/a\u003E.\u0026nbsp;\u003C\/p\u003E\u003Ch2\u003EHow does this affect my Platform.sh hosted website?\u0026nbsp;\u003C\/h2\u003E\u003Cp\u003E\u003Ca href=\u0022https:\/\/platform.sh\/\u0022 target=\u0022_blank\u0022\u003EPlatform.sh\u003C\/a\u003E scales its web hosting services based on resources (CPU, Memory, Storage, etc.). This is a more traditional approach, but costs will still rise as additional website traffic requires more resources to respond efficiently.\u003C\/p\u003E\u003Ch2\u003EStrategies to Improve Efficiency and Reduce Expenses\u003C\/h2\u003E\u003Cp\u003EIf you suspect AI crawler traffic is affecting your website\u0027s performance and cost, consider these strategies to improve efficiency and reduce expenses. Implementing advanced security measures like Cloudflare\u0027s WAF and optimizing your caching settings are proactive steps toward maintaining a robust and cost-effective online presence.\u003C\/p\u003E\u003Cp\u003EFor more detailed guidance on configuring these solutions or if you need personalized assistance, feel free to reach out to our team at Inclind. Our expertise in \u003Ca href=\u0022\/solutions\/web-hosting\u0022\u003Eweb hosting\u003C\/a\u003E and \u003Ca href=\u0022\/support\u0022\u003Emaintenance\u003C\/a\u003E, coupled with our commitment to innovative and sustainable digital solutions, ensures that your site performs well and remains cost-efficient in the face of evolving digital challenges.\u003C\/p\u003E\u003Cp\u003EIn the dynamic terrain of web development and hosting, staying proactive with technological shifts and obstacles is pivotal to your operational achievements. By controlling how bots interact with your site and optimizing your server\u0027s response to increased traffic, you can safeguard your digital assets while keeping costs in check.\u003C\/p\u003E\u003Cp\u003E\u003Ca class=\u0022hover:no-underline\u0022 href=\u0022\/support\u0022\u003E\u003Cbutton class=\u0022mt-5 btn font-semibold shadow-md transition-all duration-150 ease-linear bg-primary-600 text-white hover:bg-primary-700 active:ring active:ring-primary-50 text-md px-4 py-2.5 \u0022 type=\u0022button\u0022\u003EStay proactive and let us improve your efficiency\u003C\/button\u003E\u003C\/a\u003E\u003C\/p\u003E",
                "format": "full_html",
                "processed": "\u003Cp\u003EIn today\u0027s digital-first world, AI-powered web crawlers are increasingly traversing sites across the internet. While these bots play a crucial role in tasks like indexing for search engines and automating data collection, they also significantly increase server resource consumption. This surge can lead to higher operational costs for website owners, particularly in web hosting.\u003C\/p\u003E\n\u003Carticle class=\u0022media media--type-image media--view-mode-optimized\u0022\u003E\n  \n      \n            \u003Cdiv class=\u0022field field--name-thumbnail field--type-image field--label-hidden field__item\u0022\u003E  \u003Cimg loading=\u0022lazy\u0022 src=\u0022https:\/\/backend.inclind.com\/sites\/default\/files\/styles\/content_full\/public\/media\/images\/2024-05\/ai-crawlers.png.webp?itok=4aD6uUqd\u0022 width=\u0022800\u0022 height=\u0022316\u0022 alt=\u0022AI Crawlers \u0022 class=\u0022image-style-content-full\u0022\u003E\n\n\n\u003C\/div\u003E\n      \n  \u003C\/article\u003E\n\u003Ch2\u003EThe Impact of AI Crawlers\u003C\/h2\u003E\n\u003Cp\u003EAI crawlers scan websites systematically, consuming bandwidth and processing power. This heightened activity not only strains servers but can also slow down the user experience for human visitors. For sites with limited hosting plans, this can translate into overage fees or upgrading to more expensive hosting options to accommodate the increased load.\u003C\/p\u003E\n\u003Cp\u003EFortunately, there are effective strategies to mitigate these impacts without compromising your site\u0027s visibility or functionality:\u003C\/p\u003E\n\u003Ch2\u003EMitigating Unwanted Bot Traffic\u003C\/h2\u003E\n\u003Col\u003E\n\u003Cli\u003E\u003Cstrong\u003EEnhancing Caching Configurations:\u003C\/strong\u003E\u003Cbr\u003EFor websites running on popular CMS platforms like \u003Ca href=\u0022\/services\/wordpress-websites\u0022\u003EWordPress\u003C\/a\u003E or \u003Ca href=\u0022\/services\/drupal-websites\u0022\u003EDrupal\u003C\/a\u003E, optimizing the caching configuration can significantly reduce server load. Effective caching ensures that static parts of your website are served to visitors without repeatedly processing the same requests. This is particularly useful during high-traffic periods induced by bot activity.\u003C\/li\u003E\n\u003Cli\u003E\u003Cstrong\u003EWAF (Web Application Firewall):\u003C\/strong\u003E\u003Cbr\u003ECloudflare or Fastly provides WAF services, offering a solution that allows website owners to block or rate limit bots based on their category. This selective filtering ensures that only beneficial bots can access your site, reducing unnecessary load.\u003C\/li\u003E\n\u003Cli\u003E\u003Cstrong\u003ECloudflare\u0027s Super Bot Fight Mode:\u003C\/strong\u003E\u003Cbr\u003ECloudflare\u0027s Super Bot Fight Mode offers robust protection against malicious bots by differentiating between harmful and beneficial bot traffic on your website. It allows site owners to block, challenge, or manage bots based on their behavior, providing a dashboard for detailed insights into bot activity. This feature is ideal for sites experiencing high levels of unwanted bot traffic, efficiently enhancing site performance and security.\u003C\/li\u003E\n\u003C\/ol\u003E\n\u003Carticle class=\u0022media media--type-image media--view-mode-optimized\u0022\u003E\n  \n      \n            \u003Cdiv class=\u0022field field--name-thumbnail field--type-image field--label-hidden field__item\u0022\u003E  \u003Cimg loading=\u0022lazy\u0022 src=\u0022https:\/\/backend.inclind.com\/sites\/default\/files\/styles\/content_full\/public\/media\/images\/2024-05\/configure.png.webp?itok=dDMZiysA\u0022 width=\u0022800\u0022 height=\u0022348\u0022 alt=\u0022Configure \u0022 class=\u0022image-style-content-full\u0022\u003E\n\n\n\u003C\/div\u003E\n      \n  \u003C\/article\u003E\n\u003Ch2\u003EHow does this affect my Pantheon-hosted website?\u0026nbsp;\u003C\/h2\u003E\n\u003Cp\u003E\u003Ca href=\u0022https:\/\/pantheon.io\/\u0022 target=\u0022_blank\u0022\u003EPantheon\u003C\/a\u003E scales its web hosting based on visits. As of early 2024, it has \u003Ca href=\u0022https:\/\/pantheon.io\/blog\/enhanced-pantheon-overage-policy-traffic-limits-and-pricing\u0022 target=\u0022_blank\u0022\u003Eupdated its overage policy\u003C\/a\u003E to improve past scenarios where hosting plans were automatically upgraded if an overage occurred. It now charges $40 for each additional 10,000 site visitors over your site threshold.\u0026nbsp;\u003C\/p\u003E\n\u003Cp\u003EPantheon does offer preferred pricing and overage protections for \u003Ca href=\u0022\/clients\/non-profit-organizations\u0022\u003E501(c)3 nonprofits\u003C\/a\u003E.\u0026nbsp;\u003C\/p\u003E\n\u003Ch2\u003EHow does this affect my Platform.sh hosted website?\u0026nbsp;\u003C\/h2\u003E\n\u003Cp\u003E\u003Ca href=\u0022https:\/\/platform.sh\/\u0022 target=\u0022_blank\u0022\u003EPlatform.sh\u003C\/a\u003E scales its web hosting services based on resources (CPU, Memory, Storage, etc.). This is a more traditional approach, but costs will still rise as additional website traffic requires more resources to respond efficiently.\u003C\/p\u003E\n\u003Ch2\u003EStrategies to Improve Efficiency and Reduce Expenses\u003C\/h2\u003E\n\u003Cp\u003EIf you suspect AI crawler traffic is affecting your website\u0027s performance and cost, consider these strategies to improve efficiency and reduce expenses. Implementing advanced security measures like Cloudflare\u0027s WAF and optimizing your caching settings are proactive steps toward maintaining a robust and cost-effective online presence.\u003C\/p\u003E\n\u003Cp\u003EFor more detailed guidance on configuring these solutions or if you need personalized assistance, feel free to reach out to our team at Inclind. Our expertise in \u003Ca href=\u0022\/solutions\/web-hosting\u0022\u003Eweb hosting\u003C\/a\u003E and \u003Ca href=\u0022\/support\u0022\u003Emaintenance\u003C\/a\u003E, coupled with our commitment to innovative and sustainable digital solutions, ensures that your site performs well and remains cost-efficient in the face of evolving digital challenges.\u003C\/p\u003E\n\u003Cp\u003EIn the dynamic terrain of web development and hosting, staying proactive with technological shifts and obstacles is pivotal to your operational achievements. By controlling how bots interact with your site and optimizing your server\u0027s response to increased traffic, you can safeguard your digital assets while keeping costs in check.\u003C\/p\u003E\n\u003Cp\u003E\u003Ca class=\u0022hover:no-underline\u0022 href=\u0022\/support\u0022\u003E\u003Cbutton class=\u0022mt-5 btn font-semibold shadow-md transition-all duration-150 ease-linear bg-primary-600 text-white hover:bg-primary-700 active:ring active:ring-primary-50 text-md px-4 py-2.5 \u0022 type=\u0022button\u0022\u003EStay proactive and let us improve your efficiency\u003C\/button\u003E\u003C\/a\u003E\u003C\/p\u003E\n",
                "summary": ""
            },
            "articleType": "news",
            "author": "Shaun Tyndall",
            "authorTitle": null,
            "challenge": null,
            "introText": null,
            "location": null,
            "f_metatag": null,
            "publishDate": "2024-05-03",
            "result": null,
            "solution": null,
            "subheader1": null,
            "subheader2": null
        },
        "relationships": {
            "contentType": {
                "data": null,
                "links": {
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/contentType?resourceVersion=id%3A568"
                    }
                }
            },
            "category": {
                "data": {
                    "type": "catTerm",
                    "id": "e7befc73-77f0-46c9-aedf-608856a6a450",
                    "meta": {
                        "drupal_internal__target_id": 52
                    }
                },
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/category?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/category?resourceVersion=id%3A568"
                    }
                }
            },
            "companyRef": {
                "data": null,
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/companyRef?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/companyRef?resourceVersion=id%3A568"
                    }
                }
            },
            "image": {
                "data": {
                    "type": "images",
                    "id": "682536b8-90fd-4b1c-8865-59a96d4e1290",
                    "meta": {
                        "drupal_internal__target_id": 844
                    }
                },
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/image?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/image?resourceVersion=id%3A568"
                    }
                }
            },
            "slides": {
                "data": [],
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/slides?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/slides?resourceVersion=id%3A568"
                    }
                }
            },
            "teamMember": {
                "data": [
                    {
                        "type": "node--person",
                        "id": "87407245-73a3-43f8-9f44-9e70f40a0e1a",
                        "meta": {
                            "drupal_internal__target_id": 120
                        }
                    }
                ],
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/teamMember?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/teamMember?resourceVersion=id%3A568"
                    }
                }
            },
            "tags": {
                "data": [],
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/tags?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/tags?resourceVersion=id%3A568"
                    }
                }
            },
            "topic": {
                "data": [
                    {
                        "type": "topicTerm",
                        "id": "9af72271-b177-42c4-b243-82493ddd5393",
                        "meta": {
                            "drupal_internal__target_id": 20
                        }
                    }
                ],
                "links": {
                    "related": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/topic?resourceVersion=id%3A568"
                    },
                    "self": {
                        "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989\/relationships\/topic?resourceVersion=id%3A568"
                    }
                }
            }
        }
    },
    "links": {
        "self": {
            "href": "https:\/\/backend.inclind.com\/jsonapi\/articles\/4e0ecfed-dacb-4360-8e7d-8e9b11149989?resourceVersion=id%3A568"
        }
    }
}