From e02625c36d1e30890b90b9a9acb73c4d64f99fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20L=C3=B8vgaard?= Date: Mon, 21 Sep 2026 10:21:54 +0200 Subject: [PATCH 1/2] Replace Psalm with PHPStan, require PHP 8.1 and Symfony ^6.4 || ^7.4 || ^8.0 - Replace Psalm with PHPStan (level max) and list the code quality tools directly instead of requiring setono/code-quality-pack, since 2.x pulls in Psalm and 3.x requires PHP 8.2 - Fix PHPStan errors: explicit nullable parameter types (implicitly nullable parameters are deprecated in PHP 8.4), no short ternary, and more precise docblock types - Require PHP >=8.1 and Symfony ^6.4 || ^7.4 || ^8.0 - Convert services.xml to services.php, since the XML configuration format is deprecated in Symfony 7.4 - Upgrade to PHPUnit 10.5 (needed by matthiasnoback/symfony-dependency-injection-test for Symfony 8) and allow infection 0.32 (needed for Symfony 8) - Migrate ecs.php to ECSConfig, which newer ECS versions require - Update the CI matrix to PHP 8.1-8.4 and Symfony 6.4, 7.4 and 8.0 --- .gitattributes | 2 +- .github/workflows/build.yaml | 121 +++++------------- .gitignore | 2 +- composer.json | 36 ++++-- ecs.php | 9 +- phpstan.neon.dist | 6 + phpunit.xml.dist | 6 +- psalm.xml | 32 ----- src/BotDetector/BotDetector.php | 6 +- src/BotDetector/BotDetectorInterface.php | 2 +- .../SetonoBotDetectionExtension.php | 9 +- src/Resources/config/services.php | 45 +++++++ src/Resources/config/services.xml | 44 ------- src/Twig/Runtime.php | 2 +- tests/BotDetector/BotDetectorTest.php | 8 +- tests/Twig/ExtensionTest.php | 12 +- 16 files changed, 144 insertions(+), 198 deletions(-) create mode 100644 phpstan.neon.dist delete mode 100644 psalm.xml create mode 100644 src/Resources/config/services.php delete mode 100644 src/Resources/config/services.xml diff --git a/.gitattributes b/.gitattributes index cc4a94c..dfe35a3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,5 +7,5 @@ /ecs.php export-ignore /infection.json.dist export-ignore /phpunit.xml.dist export-ignore -/psalm.xml export-ignore +/phpstan.neon.dist export-ignore /README.md export-ignore diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 790acbd..78a5c02 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,7 +15,7 @@ jobs: strategy: matrix: php-version: - - "7.4" + - "8.1" dependencies: - "highest" @@ -53,49 +53,32 @@ jobs: strategy: matrix: php-version: - - "7.4" - - "8.0" - "8.1" - "8.2" - "8.3" + - "8.4" dependencies: - "lowest" - "highest" symfony: - - "~5.4.0" - "~6.4.0" - - "~7.0.0" - - "~7.1.0" - - "~7.2.0" + - "~7.4.0" + - "~8.0.0" exclude: - # PHP 7.4 - - php-version: "7.4" - symfony: "~6.4.0" - - php-version: "7.4" - symfony: "~7.0.0" - - php-version: "7.4" - symfony: "~7.1.0" - - php-version: "7.4" - symfony: "~7.2.0" - # PHP 8.0 - - php-version: "8.0" - symfony: "~6.4.0" - - php-version: "8.0" - symfony: "~7.0.0" - - php-version: "8.0" - symfony: "~7.1.0" - - php-version: "8.0" - symfony: "~7.2.0" # PHP 8.1 - php-version: "8.1" - symfony: "~7.0.0" - - php-version: "8.1" - symfony: "~7.1.0" + symfony: "~7.4.0" - php-version: "8.1" - symfony: "~7.2.0" + symfony: "~8.0.0" + # PHP 8.2 + - php-version: "8.2" + symfony: "~8.0.0" + # PHP 8.3 + - php-version: "8.3" + symfony: "~8.0.0" steps: - name: "Checkout" @@ -127,49 +110,32 @@ jobs: strategy: matrix: php-version: - - "7.4" - - "8.0" - "8.1" - "8.2" - "8.3" + - "8.4" dependencies: - "lowest" - "highest" symfony: - - "~5.4.0" - "~6.4.0" - - "~7.0.0" - - "~7.1.0" - - "~7.2.0" + - "~7.4.0" + - "~8.0.0" exclude: - # PHP 7.4 - - php-version: "7.4" - symfony: "~6.4.0" - - php-version: "7.4" - symfony: "~7.0.0" - - php-version: "7.4" - symfony: "~7.1.0" - - php-version: "7.4" - symfony: "~7.2.0" - # PHP 8.0 - - php-version: "8.0" - symfony: "~6.4.0" - - php-version: "8.0" - symfony: "~7.0.0" - - php-version: "8.0" - symfony: "~7.1.0" - - php-version: "8.0" - symfony: "~7.2.0" # PHP 8.1 - php-version: "8.1" - symfony: "~7.0.0" + symfony: "~7.4.0" - php-version: "8.1" - symfony: "~7.1.0" - - php-version: "8.1" - symfony: "~7.2.0" + symfony: "~8.0.0" + # PHP 8.2 + - php-version: "8.2" + symfony: "~8.0.0" + # PHP 8.3 + - php-version: "8.3" + symfony: "~8.0.0" steps: - name: "Checkout" @@ -191,7 +157,7 @@ jobs: dependency-versions: "${{ matrix.dependencies }}" - name: "Static analysis" - run: "vendor/bin/psalm --php-version=${{ matrix.php-version }}" + run: "vendor/bin/phpstan analyse --no-progress" unit-tests: name: "Unit tests" @@ -201,49 +167,32 @@ jobs: strategy: matrix: php-version: - - "7.4" - - "8.0" - "8.1" - "8.2" - "8.3" + - "8.4" dependencies: - "lowest" - "highest" symfony: - - "~5.4.0" - "~6.4.0" - - "~7.0.0" - - "~7.1.0" - - "~7.2.0" + - "~7.4.0" + - "~8.0.0" exclude: - # PHP 7.4 - - php-version: "7.4" - symfony: "~6.4.0" - - php-version: "7.4" - symfony: "~7.0.0" - - php-version: "7.4" - symfony: "~7.1.0" - - php-version: "7.4" - symfony: "~7.2.0" - # PHP 8.0 - - php-version: "8.0" - symfony: "~6.4.0" - - php-version: "8.0" - symfony: "~7.0.0" - - php-version: "8.0" - symfony: "~7.1.0" - - php-version: "8.0" - symfony: "~7.2.0" # PHP 8.1 - php-version: "8.1" - symfony: "~7.0.0" - - php-version: "8.1" - symfony: "~7.1.0" + symfony: "~7.4.0" - php-version: "8.1" - symfony: "~7.2.0" + symfony: "~8.0.0" + # PHP 8.2 + - php-version: "8.2" + symfony: "~8.0.0" + # PHP 8.3 + - php-version: "8.3" + symfony: "~8.0.0" steps: - name: "Checkout" diff --git a/.gitignore b/.gitignore index 39a92f2..cc81ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ /vendor/ /composer.lock -/.phpunit.result.cache +/.phpunit.cache/ /build/ diff --git a/composer.json b/composer.json index f691bfc..a36d769 100644 --- a/composer.json +++ b/composer.json @@ -10,25 +10,32 @@ } ], "require": { - "php": ">=7.4", - "symfony/config": "^5.4 || ^6.4 || ^7.0", - "symfony/dependency-injection": "^5.4 || ^6.4 || ^7.0", - "symfony/http-foundation": "^5.4 || ^6.4 || ^7.0", - "symfony/http-kernel": "^5.4 || ^6.4 || ^7.0", + "php": ">=8.1", + "symfony/config": "^6.4 || ^7.4 || ^8.0", + "symfony/dependency-injection": "^6.4 || ^7.4 || ^8.0", + "symfony/http-foundation": "^6.4 || ^7.4 || ^8.0", + "symfony/http-kernel": "^6.4 || ^7.4 || ^8.0", "twig/twig": "^2.14 || ^3.4" }, "require-dev": { - "infection/infection": "^0.26.6 || ^0.27.0", + "ergebnis/composer-normalize": "^2.42", + "infection/infection": "^0.27.0 || ^0.32.0", + "jangregor/phpstan-prophecy": "^2.0", "matomo/device-detector": "^5.0 || ^6.0", - "matthiasnoback/symfony-dependency-injection-test": "^4.3 || ^5.1", + "matthiasnoback/symfony-dependency-injection-test": "^5.1 || ^6.3", "nette/php-generator": "^3.6 || ^4.1", "phpspec/prophecy-phpunit": "^2.0", - "phpunit/phpunit": "^9.6", - "psalm/plugin-phpunit": "^0.18", - "setono/code-quality-pack": "^2.9", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-phpunit": "^2.0", + "phpstan/phpstan-strict-rules": "^2.0", + "phpstan/phpstan-webmozart-assert": "^2.0", + "phpunit/phpunit": "^10.5", + "rector/rector": "^2.0", "shipmonk/composer-dependency-analyser": "^1.8", - "symfony/console": "^5.4 || ^6.4 || ^7.0", - "symfony/yaml": "^5.4 || ^6.4 || ^7.0", + "sylius-labs/coding-standard": "^4.1.1", + "symfony/console": "^6.4 || ^7.4 || ^8.0", + "symfony/yaml": "^6.4 || ^7.4 || ^8.0", "webmozart/assert": "^1.11" }, "prefer-stable": true, @@ -46,12 +53,13 @@ "allow-plugins": { "dealerdirect/phpcodesniffer-composer-installer": false, "ergebnis/composer-normalize": true, - "infection/extension-installer": true + "infection/extension-installer": true, + "phpstan/extension-installer": true }, "sort-packages": true }, "scripts": { - "analyse": "psalm", + "analyse": "phpstan analyse", "check-style": "ecs check", "fix-style": "ecs check --fix", "phpunit": "phpunit" diff --git a/ecs.php b/ecs.php index e2c5168..6397e61 100644 --- a/ecs.php +++ b/ecs.php @@ -2,12 +2,11 @@ declare(strict_types=1); -use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; -use Symplify\EasyCodingStandard\ValueObject\Option; +use Symplify\EasyCodingStandard\Config\ECSConfig; -return static function (ContainerConfigurator $configurator): void { - $configurator->import('vendor/sylius-labs/coding-standard/ecs.php'); - $configurator->parameters()->set(Option::PATHS, [ +return static function (ECSConfig $config): void { + $config->import('vendor/sylius-labs/coding-standard/ecs.php'); + $config->paths([ 'src', 'tests', ]); diff --git a/phpstan.neon.dist b/phpstan.neon.dist new file mode 100644 index 0000000..882dad7 --- /dev/null +++ b/phpstan.neon.dist @@ -0,0 +1,6 @@ +parameters: + level: max + paths: + - bin + - src + - tests diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 0b06dc8..60313bd 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -1,11 +1,11 @@ - + xsi:noNamespaceSchemaLocation="vendor/phpunit/phpunit/phpunit.xsd" colors="true" cacheDirectory=".phpunit.cache"> + src/ - + tests diff --git a/psalm.xml b/psalm.xml deleted file mode 100644 index b45472e..0000000 --- a/psalm.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/BotDetector/BotDetector.php b/src/BotDetector/BotDetector.php index acf0656..34dcacf 100644 --- a/src/BotDetector/BotDetector.php +++ b/src/BotDetector/BotDetector.php @@ -20,7 +20,7 @@ final class BotDetector implements BotDetectorInterface /** * @param list|null $popular */ - public function __construct(RequestStack $requestStack, array $popular = null) + public function __construct(RequestStack $requestStack, ?array $popular = null) { $this->requestStack = $requestStack; if (null === $popular) { @@ -45,13 +45,13 @@ public function isBot(string $userAgent): bool if (!isset($this->cache[$userAgent])) { $minimalRegex = '#' . implode('|', $this->popular) . '#'; $minimalMatch = preg_match($minimalRegex, $userAgent) === 1; - $this->cache[$userAgent] = $minimalMatch ?: preg_match(Bots::REGEX, $userAgent) === 1; + $this->cache[$userAgent] = $minimalMatch || preg_match(Bots::REGEX, $userAgent) === 1; } return $this->cache[$userAgent]; } - public function isBotRequest(Request $request = null): bool + public function isBotRequest(?Request $request = null): bool { $request = $request ?? $this->requestStack->getMainRequest(); if (null === $request) { diff --git a/src/BotDetector/BotDetectorInterface.php b/src/BotDetector/BotDetectorInterface.php index 178fecf..f2f71f1 100644 --- a/src/BotDetector/BotDetectorInterface.php +++ b/src/BotDetector/BotDetectorInterface.php @@ -13,5 +13,5 @@ public function isBot(string $userAgent): bool; /** * Uses the current main request if $request is null */ - public function isBotRequest(Request $request = null): bool; + public function isBotRequest(?Request $request = null): bool; } diff --git a/src/DependencyInjection/SetonoBotDetectionExtension.php b/src/DependencyInjection/SetonoBotDetectionExtension.php index a7245bc..f5298f7 100644 --- a/src/DependencyInjection/SetonoBotDetectionExtension.php +++ b/src/DependencyInjection/SetonoBotDetectionExtension.php @@ -7,13 +7,16 @@ use Symfony\Component\Config\FileLocator; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Extension\Extension; -use Symfony\Component\DependencyInjection\Loader\XmlFileLoader; +use Symfony\Component\DependencyInjection\Loader\PhpFileLoader; final class SetonoBotDetectionExtension extends Extension { + /** + * @param array> $configs + */ public function load(array $configs, ContainerBuilder $container): void { - $loader = new XmlFileLoader($container, new FileLocator(__DIR__ . '/../Resources/config')); - $loader->load('services.xml'); + $loader = new PhpFileLoader($container, new FileLocator(__DIR__ . '/../Resources/config')); + $loader->load('services.php'); } } diff --git a/src/Resources/config/services.php b/src/Resources/config/services.php new file mode 100644 index 0000000..ece2284 --- /dev/null +++ b/src/Resources/config/services.php @@ -0,0 +1,45 @@ +parameters()->set('setono_bot_detection.popular_bots', [ + 'Googlebot', + 'Bingbot', + 'Yahoo! Slurp', + 'DuckDuckBot', + 'Baiduspider', + 'YandexBot', + 'facebookexternalhit', + 'facebookcatalog', + 'ia_archiver', + ]); + + $services = $container->services(); + + $services->alias(BotDetectorInterface::class, 'setono_bot_detection.bot_detector.default'); + + $services->set('setono_bot_detection.bot_detector.default', BotDetector::class) + ->args([ + service('request_stack'), + param('setono_bot_detection.popular_bots'), + ]); + + $services->set('setono_bot_detection.twig.extension', Extension::class) + ->tag('twig.extension'); + + $services->set('setono_bot_detection.twig.runtime', Runtime::class) + ->args([service('setono_bot_detection.bot_detector.default')]) + ->tag('twig.runtime'); +}; diff --git a/src/Resources/config/services.xml b/src/Resources/config/services.xml deleted file mode 100644 index aa0a616..0000000 --- a/src/Resources/config/services.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - Googlebot - Bingbot - Yahoo! Slurp - DuckDuckBot - Baiduspider - YandexBot - facebookexternalhit - facebookcatalog - ia_archiver - - - - - - - - %setono_bot_detection.popular_bots% - - - - - - - - - - - - - diff --git a/src/Twig/Runtime.php b/src/Twig/Runtime.php index 53c96ac..aea5f1c 100644 --- a/src/Twig/Runtime.php +++ b/src/Twig/Runtime.php @@ -17,7 +17,7 @@ public function __construct(BotDetectorInterface $botDetector) $this->botDetector = $botDetector; } - public function isBotRequest(Request $request = null): bool + public function isBotRequest(?Request $request = null): bool { return $this->botDetector->isBotRequest($request); } diff --git a/tests/BotDetector/BotDetectorTest.php b/tests/BotDetector/BotDetectorTest.php index d9f25e8..bdeaeab 100644 --- a/tests/BotDetector/BotDetectorTest.php +++ b/tests/BotDetector/BotDetectorTest.php @@ -48,6 +48,7 @@ public function it_returns_false_if_no_request_is_provided_and_the_request_does_ /** * @dataProvider getBots + * * @test */ public function it_detects_common_bots(string $userAgent): void @@ -58,6 +59,7 @@ public function it_detects_common_bots(string $userAgent): void /** * @dataProvider getHumanUserAgents + * * @test */ public function it_does_not_detect_common_human_user_agents(string $userAgent): void @@ -69,8 +71,9 @@ public function it_does_not_detect_common_human_user_agents(string $userAgent): /** * @return iterable> */ - public function getBots(): iterable + public static function getBots(): iterable { + /** @var list $bots */ $bots = require __DIR__ . '/../data/bots.php'; foreach ($bots as $bot) { yield [$bot]; @@ -80,8 +83,9 @@ public function getBots(): iterable /** * @return iterable> */ - public function getHumanUserAgents(): iterable + public static function getHumanUserAgents(): iterable { + /** @var list $humans */ $humans = require __DIR__ . '/../data/humans.php'; foreach ($humans as $human) { yield [$human]; diff --git a/tests/Twig/ExtensionTest.php b/tests/Twig/ExtensionTest.php index 0ce8035..e88de09 100644 --- a/tests/Twig/ExtensionTest.php +++ b/tests/Twig/ExtensionTest.php @@ -31,7 +31,7 @@ public function isBot(string $userAgent): bool return true; } - public function isBotRequest(Request $request = null): bool + public function isBotRequest(?Request $request = null): bool { return true; } @@ -51,8 +51,16 @@ public function getExtensions(): array ]; } - protected function getFixturesDir(): string + protected static function getFixturesDirectory(): string { return __DIR__ . '/Fixtures/'; } + + /** + * Used by twig/twig < 3.13 + */ + protected function getFixturesDir(): string + { + return self::getFixturesDirectory(); + } } From 789a98a0ed58a342e05b5fe498556ade4f942db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20L=C3=B8vgaard?= Date: Mon, 21 Sep 2026 10:28:26 +0200 Subject: [PATCH 2/2] Update the popular bots list and match it case-insensitively The popular bots list is a fast path checked before the full bot regex. It is now the 20 highest-volume crawlers based on Cloudflare Radar data (search engines, AI crawlers, SEO crawlers and Meta's fetchers). Yahoo! Slurp, ia_archiver and facebookcatalog are dropped: Yahoo search is served by Bing, Alexa was retired in 2022 and facebookcatalog is low volume. The list is now matched case-insensitively. Previously the "Bingbot" entry never matched Bing's real user agent, which uses "bingbot". --- src/BotDetector/BotDetector.php | 25 +++++++++++++++------ src/Resources/config/services.php | 25 +++++++++++++++------ tests/BotDetector/BotDetectorTest.php | 9 ++++++++ tests/data/bots.php | 31 +++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/BotDetector/BotDetector.php b/src/BotDetector/BotDetector.php index 34dcacf..9f3805f 100644 --- a/src/BotDetector/BotDetector.php +++ b/src/BotDetector/BotDetector.php @@ -26,14 +26,25 @@ public function __construct(RequestStack $requestStack, ?array $popular = null) if (null === $popular) { $popular = [ 'Googlebot', - 'Bingbot', - 'Yahoo! Slurp', - 'DuckDuckBot', - 'Baiduspider', + 'meta-externalagent', + 'GPTBot', + 'ClaudeBot', + 'bingbot', + 'Amazonbot', + 'GoogleOther', 'YandexBot', + 'Bytespider', + 'AhrefsBot', + 'Applebot', + 'SemrushBot', + 'ChatGPT-User', + 'OAI-SearchBot', + 'PerplexityBot', + 'meta-webindexer', 'facebookexternalhit', - 'facebookcatalog', - 'ia_archiver', + 'Baiduspider', + 'PetalBot', + 'DuckDuckBot', ]; } @@ -43,7 +54,7 @@ public function __construct(RequestStack $requestStack, ?array $popular = null) public function isBot(string $userAgent): bool { if (!isset($this->cache[$userAgent])) { - $minimalRegex = '#' . implode('|', $this->popular) . '#'; + $minimalRegex = '#' . implode('|', $this->popular) . '#i'; $minimalMatch = preg_match($minimalRegex, $userAgent) === 1; $this->cache[$userAgent] = $minimalMatch || preg_match(Bots::REGEX, $userAgent) === 1; } diff --git a/src/Resources/config/services.php b/src/Resources/config/services.php index ece2284..9c4716a 100644 --- a/src/Resources/config/services.php +++ b/src/Resources/config/services.php @@ -12,18 +12,29 @@ return static function (ContainerConfigurator $container): void { /** * Use this collection to add popular bots to the bot detector. This will speed up detection when the bot is one of these. - * These are regexes that should match the user agent of the respective bot. + * These are regexes that should match the user agent of the respective bot. They are matched case-insensitively. */ $container->parameters()->set('setono_bot_detection.popular_bots', [ 'Googlebot', - 'Bingbot', - 'Yahoo! Slurp', - 'DuckDuckBot', - 'Baiduspider', + 'meta-externalagent', + 'GPTBot', + 'ClaudeBot', + 'bingbot', + 'Amazonbot', + 'GoogleOther', 'YandexBot', + 'Bytespider', + 'AhrefsBot', + 'Applebot', + 'SemrushBot', + 'ChatGPT-User', + 'OAI-SearchBot', + 'PerplexityBot', + 'meta-webindexer', 'facebookexternalhit', - 'facebookcatalog', - 'ia_archiver', + 'Baiduspider', + 'PetalBot', + 'DuckDuckBot', ]); $services = $container->services(); diff --git a/tests/BotDetector/BotDetectorTest.php b/tests/BotDetector/BotDetectorTest.php index bdeaeab..373711e 100644 --- a/tests/BotDetector/BotDetectorTest.php +++ b/tests/BotDetector/BotDetectorTest.php @@ -46,6 +46,15 @@ public function it_returns_false_if_no_request_is_provided_and_the_request_does_ self::assertFalse($botDetector->isBotRequest()); } + /** + * @test + */ + public function it_matches_popular_bots_case_insensitively(): void + { + $botDetector = new BotDetector(new RequestStack(), ['Qwertz']); + self::assertTrue($botDetector->isBot('Mozilla/5.0 (compatible; qwertz/1.0)')); + } + /** * @dataProvider getBots * diff --git a/tests/data/bots.php b/tests/data/bots.php index 607cc69..f3ec924 100644 --- a/tests/data/bots.php +++ b/tests/data/bots.php @@ -14,9 +14,38 @@ 'Mediapartners-Google', 'AdsBot-Google (+http://www.google.com/adsbot.html)', 'AdsBot-Google-Mobile-Apps', + 'Mozilla/5.0 (compatible; GoogleOther)', // bing 'Mozilla/5.0 (compatible Bingbot/2.0 +http://www.bing.com/bingbot.htm)', + 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', + + // openai + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)', + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot', + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot', + + // anthropic + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', + + // perplexity + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', + + // amazon + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36', + + // apple + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)', + + // bytedance + 'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)', + + // huawei + 'Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)', + + // seo + 'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)', + 'Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)', // yahoo 'Mozilla/5.0 (compatible Yahoo! Slurp http://help.yahoo.com/help/us/ysearch/slurp)', @@ -45,6 +74,8 @@ 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', 'facebookexternalhit/1.1', 'facebookcatalog/1.0', + 'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)', + 'meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)', // alexa 'ia_archiver (+http://www.alexa.com/site/help/webmasters crawler@alexa.com)',