Skip to content

Tests: Fix Flaky Subscriber Checks #1183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
n7studios wants to merge 6 commits into
base: main
Choose a base branch
from
+234 −69
Open

Tests: Fix Flaky Subscriber Checks #1183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
49311f8
Tests: Wait for subscriber
n7studios Sep 8, 2026
af4387a
Tests: Log API Requests
n7studios Sep 8, 2026
76e3c52
Run specific tests
n7studios Sep 8, 2026
71902d1
Reinstate all tests
n7studios Sep 8, 2026
7929f39
Merge branch 'main' into tests-fix-check-subscriber
n7studios Sep 8, 2026
c29bf35
Revert apiCheckSubscriberDoesNotExist
n7studios Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 181 additions & 69 deletions tests/Support/Helper/KitAPI.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,91 @@
*/
class KitAPI extends \Codeception\Module
{
/**
* Installs the Kit API recorder mu-plugin, and clears any previously recorded
* requests, before each test runs.
*
* @since 3.4.1
*
* @param \Codeception\TestInterface $test Test.
*/
public function _before(\Codeception\TestInterface $test) // phpcs:ignore PSR2.Methods.MethodDeclaration.Underscore, Generic.CodeAnalysis.UnusedFunctionParameter
{
$this->getModule('lucatume\WPBrowser\Module\WPFilesystem')->haveMuPlugin(
'kit-api-recorder.php',
(string) file_get_contents(__DIR__ . '/../mu-plugins/kit-api-recorder.php') // phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents
);

$this->getModule('lucatume\WPBrowser\Module\WPDb')->haveOptionInDatabase('kit_api_log', []);
}

/**
* Returns the Kit API requests the Plugin made during this test, optionally
* filtered by method, path and email address.
*
* @since 3.4.1
*
* @param EndToEndTester $I EndToEndTester.
* @param string $method HTTP method (GET,POST,PUT,DELETE).
* @param string $path Request path, excluding the API version e.g. `subscribers`.
* @param bool|string $emailAddress Email address in the request body.
* @return array
*/
public function grabKitAPIRequests($I, $method = false, $path = false, $emailAddress = false)
{
$log = $I->grabOptionFromDatabase('kit_api_log');

if ( ! is_array($log)) {
return [];
}

return array_values(
array_filter(
$log,
function ($request) use ($method, $path, $emailAddress) {
if ($method && $request['method'] !== $method) {
return false;
}
if ($path && $request['path'] !== $path) {
return false;
}
if ($emailAddress && ( ! array_key_exists('email_address', $request['body']) || $request['body']['email_address'] !== $emailAddress )) {
return false;
}

return true;
}
)
);
}

/**
* Returns the first Kit API request the Plugin made during this test that matches
* the given method, path and email address, waiting for it to be made.
*
* @since 3.4.1
*
* @param EndToEndTester $I EndToEndTester.
* @param string $method HTTP method (GET,POST,PUT,DELETE).
* @param string $path Request path, excluding the API version e.g. `subscribers`.
* @param bool|string $emailAddress Email address in the request body.
* @return bool|array
*/
public function grabKitAPIRequest($I, $method, $path, $emailAddress = false)
{
// The request is made by WordPress when the form is submitted, which may not have
// completed when this is called e.g. when a form submits using AJAX.
return $this->retryUntil(
function () use ($I, $method, $path, $emailAddress) {
$requests = $this->grabKitAPIRequests($I, $method, $path, $emailAddress);

return count($requests) ? $requests[0] : false;
},
10,
1
);
}

/**
* Returns an encoded `state` parameter compatible with OAuth.
*
Expand Down Expand Up @@ -42,49 +127,46 @@ public function apiEncodeState($returnTo, $clientID)
/**
* Check the given email address exists as a subscriber.
*
* The Plugin's request to create the subscriber is used to determine the subscriber ID,
* as querying the API by email address is subject to eventual consistency. Querying by
* subscriber ID returns strongly consistent results.
*
* @see https://developers.kit.com/api-reference/eventual-consistency
*
* @param EndToEndTester $I EndToEndTester.
* @param string $emailAddress Email Address.
* @param string $firstName First Name (false = don't check name matches).
* @return array Subscriber.
*/
public function apiCheckSubscriberExists($I, $emailAddress, $firstName = false)
{
// Wait for the API to update.
$I->wait(3);

// Retry the API request as sometimes there's a lag before the subscriber is queryable via the API.
$results = $this->retryUntil(
function () use ($emailAddress) {
$results = $this->apiRequest(
'subscribers',
'GET',
[
'email_address' => $emailAddress,
'include_total_count' => true,

// Check all subscriber states.
'status' => 'all',
]
);
// Get the request the Plugin made to create the subscriber.
$request = $this->grabKitAPIRequest($I, 'POST', 'subscribers', $emailAddress);

// Return the results only if a subscriber was found, so
// retryUntil() will keep trying otherwise.
return ( $results['pagination']['total_count'] > 0 ) ? $results : false;
}
// Check the Plugin created the subscriber.
$I->assertNotFalse(
$request,
sprintf('The Plugin did not send a request to create the subscriber %s.', $emailAddress)
);
$I->assertLessThan(
300,
$request['code'],
sprintf('The API returned a %s response when the Plugin created the subscriber %s.', $request['code'], $emailAddress)
);

// Check at least one subscriber was returned and it matches the email address.
$I->assertNotFalse($results);
$I->assertGreaterThan(0, $results['pagination']['total_count']);
$I->assertEquals($emailAddress, $results['subscribers'][0]['email_address']);
// Fetch the subscriber by their ID, which returns strongly consistent results.
$results = $this->apiRequest('subscribers/' . $request['response']['subscriber']['id'], 'GET');

// Check the subscriber matches the email address.
$I->assertEquals($emailAddress, $results['subscriber']['email_address']);

// If defined, check that the name matches for the subscriber.
if ($firstName) {
$I->assertEquals($firstName, $results['subscribers'][0]['first_name']);
$I->assertEquals($firstName, $results['subscriber']['first_name']);
}

// Return subscriber ID.
return $results['subscribers'][0];
// Return subscriber.
return $results['subscriber'];
}

/**
Expand All @@ -99,27 +181,35 @@ function () use ($emailAddress) {
*/
public function apiCheckSubscriberHasForm($I, $subscriberID, $formID, $referrer = false)
{
// Run request.
$results = $this->apiRequest(
'forms/' . $formID . '/subscribers',
'GET',
[
// Check all subscriber states.
'status' => 'all',
]
);
// Wait for the subscriber to be assigned to the form, as list endpoints are eventually consistent.
$subscriber = $this->retryUntil(
function () use ($subscriberID, $formID) {
$results = $this->apiRequest(
'forms/' . $formID . '/subscribers',
'GET',
[
// Check all subscriber states.
'status' => 'all',
]
);

// Iterate through subscribers.
$subscriberHasForm = false;
foreach ($results['subscribers'] as $subscriber) {
if ($subscriber['id'] === $subscriberID) {
$subscriberHasForm = true;
break;
// Return the subscriber only if they're assigned to the form, so
// retryUntil() will keep trying otherwise.
foreach ($results['subscribers'] as $subscriber) {
if ( (int) $subscriber['id'] === (int) $subscriberID) {
return $subscriber;
}
}

return false;
}
}
);

// Assert if the subscriber has the form.
$this->assertTrue($subscriberHasForm);
// Assert the subscriber has the form.
$I->assertNotFalse(
$subscriber,
sprintf('Subscriber %s was not assigned to Form %s in time.', $subscriberID, $formID)
);

// If a referrer is specified, assert it matches the subscriber's referrer now.
if ($referrer) {
Expand All @@ -138,26 +228,34 @@ public function apiCheckSubscriberHasForm($I, $subscriberID, $formID, $referrer
*/
public function apiCheckSubscriberHasSequence($I, $subscriberID, $sequenceID)
{
// Run request.
$results = $this->apiRequest(
'sequences/' . $sequenceID . '/subscribers',
'GET',
[
'status' => 'all',
]
);
// Wait for the subscriber to be assigned to the sequence, as list endpoints are eventually consistent.
$subscriber = $this->retryUntil(
function () use ($subscriberID, $sequenceID) {
$results = $this->apiRequest(
'sequences/' . $sequenceID . '/subscribers',
'GET',
[
'status' => 'all',
]
);

// Iterate through subscribers.
$subscriberHasSequence = false;
foreach ($results['subscribers'] as $subscriber) {
if ($subscriber['id'] === $subscriberID) {
$subscriberHasSequence = true;
break;
// Return the subscriber only if they're assigned to the sequence, so
// retryUntil() will keep trying otherwise.
foreach ($results['subscribers'] as $subscriber) {
if ( (int) $subscriber['id'] === (int) $subscriberID) {
return $subscriber;
}
}

return false;
}
}
);

// Assert if the subscriber has the sequence.
$this->assertTrue($subscriberHasSequence);
// Assert the subscriber has the sequence.
$I->assertNotFalse(
$subscriber,
sprintf('Subscriber %s was not assigned to Sequence %s in time.', $subscriberID, $sequenceID)
);
}

/**
Expand All @@ -169,10 +267,24 @@ public function apiCheckSubscriberHasSequence($I, $subscriberID, $sequenceID)
*/
public function apiCheckSubscriberHasTag($I, $subscriberID, $tagID)
{
// Run request.
$results = $this->apiRequest(
'subscribers/' . $subscriberID . '/tags',
'GET'
// Wait for the tag to be assigned to the subscriber, as list endpoints are eventually consistent.
$results = $this->retryUntil(
function () use ($subscriberID) {
$results = $this->apiRequest(
'subscribers/' . $subscriberID . '/tags',
'GET'
);

// Return the results only if a tag is assigned, so
// retryUntil() will keep trying otherwise.
return count($results['tags']) ? $results : false;

@ciccio-kit ciccio-kit Sep 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The retry stops as soon as the subscriber has any tag, but line 291 then asserts the first tag is $tagID, so I think that if another tag lands first, you exit the retry early and fail immediately.

}
);

// Assert the subscriber has a tag.
$I->assertNotFalse(
$results,
sprintf('Subscriber %s was not assigned Tag %s in time.', $subscriberID, $tagID)
);

// Confirm the tag has been assigned to the subscriber.
Expand Down Expand Up @@ -316,8 +428,8 @@ public function apiRequest($endpoint, $method = 'GET', $params = array())
[
'headers' => [
'Authorization' => 'Bearer ' . $_ENV['CONVERTKIT_OAUTH_ACCESS_TOKEN'],
'timeout' => 5,
],
'timeout' => 5,
]
);
break;
Expand All @@ -331,8 +443,8 @@ public function apiRequest($endpoint, $method = 'GET', $params = array())
'Accept' => 'application/json',
'Content-Type' => 'application/json; charset=utf-8',
'Authorization' => 'Bearer ' . $_ENV['CONVERTKIT_OAUTH_ACCESS_TOKEN'],
'timeout' => 5,
],
'timeout' => 5,
'body' => (string) json_encode($params), // phpcs:ignore WordPress.WP.AlternativeFunctions
]
);
Expand Down
53 changes: 53 additions & 0 deletions tests/Support/mu-plugins/kit-api-recorder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
<?php
/**
* Plugin Name: Kit API Recorder
* Description: Records Kit API requests and responses in an option, for end to end tests to assert against.
*
* @package ConvertKit
* @author ConvertKit
*/

// Record Kit API requests and responses.
add_action(
'http_api_debug',
function ( $response, $context, $transport, $parsed_args, $url ) { // phpcs:ignore Generic.CodeAnalysis.UnusedFunctionParameter

// Bail if this isn't a request to the Kit API.
if ( strpos( $url, 'https://api.kit.com/' ) !== 0 ) {
return;
}

// Build the request path, excluding the API version and any query parameters.
$path = (string) wp_parse_url( $url, PHP_URL_PATH );
$path = ltrim( str_replace( '/v4/', '', $path ), '/' );
$query = array();
parse_str( (string) wp_parse_url( $url, PHP_URL_QUERY ), $query );

// Decode the request body, which is JSON encoded for POST, PUT and DELETE requests.
$body = array();
if ( ! empty( $parsed_args['body'] ) ) {
$body = is_string( $parsed_args['body'] ) ? json_decode( $parsed_args['body'], true ) : $parsed_args['body'];
}

// Fetch the existing log.
$log = get_option( 'kit_api_log', array() );
if ( ! is_array( $log ) ) {
$log = array();
}

// Append this request and its response to the log.
$log[] = array(
'method' => isset( $parsed_args['method'] ) ? $parsed_args['method'] : 'GET',
'path' => $path,
'query' => $query,
'body' => is_array( $body ) ? $body : array(),
'code' => is_wp_error( $response ) ? 0 : (int) wp_remote_retrieve_response_code( $response ),
'error' => is_wp_error( $response ) ? $response->get_error_message() : '',
'response' => is_wp_error( $response ) ? array() : (array) json_decode( wp_remote_retrieve_body( $response ), true ),
);

update_option( 'kit_api_log', $log, false );
},
10,
5
);