Merge pull request #125 from Automattic/fix/37-events-cache

Chunk events queue for cacheability

includes/class-events-store.php

@@ -252,7 +252,7 @@ class Events_Store extends Singleton {
 	 */
 	public function get_option() {
 		// Use cached value when available.
-		$cached_option = wp_cache_get( self::CACHE_KEY, null, true );
+		$cached_option = $this->get_cached_option();
 
 		if ( false !== $cached_option ) {
 			return $cached_option;
@@ -312,7 +312,7 @@ class Events_Store extends Singleton {
 		uksort( $cron_array, 'strnatcasecmp' );
 
 		// Cache the results.
-		wp_cache_set( self::CACHE_KEY, $cron_array, null, 1 * \HOUR_IN_SECONDS );
+		$this->cache_option( $cron_array );
 
 		return $cron_array;
 	}
@@ -680,6 +680,109 @@ class Events_Store extends Singleton {
 		return $differences;
 	}
 
+	/**
+	 * Retrieve cron option from cache
+	 *
+	 * @return array|false
+	 */
+	private function get_cached_option() {
+		$cache_details = wp_cache_get( self::CACHE_KEY, null, true );
+
+		if ( ! is_array( $cache_details ) ) {
+			return false;
+		}
+
+		// Single bucket.
+		if ( isset( $cache_details['version'] ) ) {
+			return $cache_details;
+		}
+
+		// Invalid data!
+		if ( ! isset( $cache_details['incrementer'] ) ) {
+			return false;
+		}
+
+		$option_flat = array();
+
+		// Restore option from cached pieces.
+		for ( $i = 1; $i <= $cache_details['buckets']; $i++ ) {
+			$cache_key    = $this->get_cache_key_for_slice( $cache_details['incrementer'], $i );
+			$cached_slice = wp_cache_get( $cache_key, null, true );
+
+			// Bail if a chunk is missing.
+			if ( ! is_array( $cached_slice ) ) {
+				return false;
+			}
+
+			$option_flat += $cached_slice;
+		}
+
+		// Something's missing, likely due to cache eviction.
+		if ( empty( $option_flat ) || count( $option_flat ) !== $cache_details['event_count'] ) {
+			return false;
+		}
+
+		return inflate_collapsed_events_array( $option_flat );
+	}
+
+	/**
+	 * Cache cron option, accommodating large versions by splitting into chunks
+	 *
+	 * @param array $option Cron option to cache.
+	 * @return bool
+	 */
+	private function cache_option( $option ) {
+		// Determine storage requirements.
+		$option_flat        = collapse_events_array( $option );
+		$option_flat_string = maybe_serialize( $option_flat );
+		$option_size        = strlen( $option_flat_string );
+		$buckets            = (int) ceil( $option_size / CACHE_BUCKET_SIZE );
+
+		// Store in single cache key.
+		if ( 1 === $buckets ) {
+			return wp_cache_set( self::CACHE_KEY, $option, null, 1 * \HOUR_IN_SECONDS );
+		}
+
+		// Too large to cache?
+		if ( $buckets > MAX_CACHE_BUCKETS ) {
+			do_action( 'a8c_cron_control_uncacheable_cron_option', $option_size, $buckets, count( $option_flat ) );
+
+			$this->flush_internal_caches();
+			return false;
+		}
+
+		$incrementer  = md5( $option_flat_string . time() );
+		$event_count  = count( $option_flat );
+		$segment_size = (int) ceil( $event_count / $buckets );
+
+		for ( $i = 1; $i <= $buckets; $i++ ) {
+			$offset    = ( $i - 1 ) * $segment_size;
+			$slice     = array_slice( $option_flat, $offset, $segment_size );
+			$cache_key = $this->get_cache_key_for_slice( $incrementer, $i );
+
+			wp_cache_set( $cache_key, $slice, null, 1 * \HOUR_IN_SECONDS );
+		}
+
+		$option = array(
+			'incrementer' => $incrementer,
+			'buckets'     => $buckets,
+			'event_count' => count( $option_flat ),
+		);
+
+		return wp_cache_set( self::CACHE_KEY, $option, null, 1 * \HOUR_IN_SECONDS );
+	}
+
+	/**
+	 * Build cache key for a given portion of a large option
+	 *
+	 * @param string $incrementor Current cache incrementor.
+	 * @param int    $slice Slice ID.
+	 * @return string
+	 */
+	private function get_cache_key_for_slice( $incrementor, $slice ) {
+		return md5( self::CACHE_KEY . $incrementor . $slice );
+	}
+
 	/**
 	 * Delete the cached representation of the cron option
 	 */

includes/constants.php

@@ -39,6 +39,28 @@ const JOB_LOCK_EXPIRY_IN_MINUTES  = 30;
 const LOCK_DEFAULT_LIMIT              = 10;
 const LOCK_DEFAULT_TIMEOUT_IN_MINUTES = 10;
 
+/**
+ * Limit on size of event cache objects
+ */
+$cache_bucket_size = \MB_IN_BYTES * 0.95;
+if ( defined( 'CRON_CONTROL_CACHE_BUCKET_SIZE' ) && is_numeric( \CRON_CONTROL_CACHE_BUCKET_SIZE ) ) {
+	$cache_bucket_size = absint( \CRON_CONTROL_CACHE_BUCKET_SIZE );
+	$cache_bucket_size = max( 256 * \KB_IN_BYTES, min( $cache_bucket_size, \TB_IN_BYTES ) );
+}
+define( __NAMESPACE__ . '\CACHE_BUCKET_SIZE', $cache_bucket_size );
+unset( $cache_bucket_size );
+
+/**
+ * Limit how many buckets can be created, to avoid cache exhaustion
+ */
+$max_cache_buckets = 5;
+if ( defined( 'CRON_CONTROL_MAX_CACHE_BUCKETS' ) && is_numeric( \CRON_CONTROL_MAX_CACHE_BUCKETS ) ) {
+	$max_cache_buckets = absint( \CRON_CONTROL_MAX_CACHE_BUCKETS );
+	$max_cache_buckets = max( 1, min( $max_cache_buckets, 250 ) );
+}
+define( __NAMESPACE__ . '\MAX_CACHE_BUCKETS', $max_cache_buckets );
+unset( $max_cache_buckets );
+
 /**
  * Consistent time format across plugin
  *

includes/utils.php

@@ -59,6 +59,41 @@ function collapse_events_array( $events, $timestamp = null ) {
 	return $collapsed_events;
 }
 
+/**
+ * Convert simplified representation of cron events array to the format WordPress expects
+ *
+ * @param array $events Flattened event list.
+ * @return array
+ */
+function inflate_collapsed_events_array( $events ) {
+	$inflated = array(
+		'version' => 2, // Core versions the cron array; without this, Core will attempt to "upgrade" the value.
+	);
+
+	if ( empty( $events ) ) {
+		return $inflated;
+	}
+
+	foreach ( $events as $event ) {
+		// Object for convenience.
+		$event = (object) $event;
+
+		// Set up where this event belongs in the overall structure.
+		if ( ! isset( $inflated[ $event->timestamp ] ) ) {
+			$inflated[ $event->timestamp ] = array();
+		}
+
+		if ( ! isset( $inflated[ $event->timestamp ][ $event->action ] ) ) {
+			$inflated[ $event->timestamp ][ $event->action ] = array();
+		}
+
+		// Store this event.
+		$inflated[ $event->timestamp ][ $event->action ][ $event->instance ] = $event->args;
+	}
+
+	return $inflated;
+}
+
 /**
  * Parse request using Core's logic
  *

tests/bootstrap.php

@@ -19,6 +19,10 @@ require_once $_tests_dir . '/includes/functions.php';
 function _manually_load_plugin() {
 	define( 'WP_CRON_CONTROL_SECRET', 'testtesttest' );
 
+	// Nonsense values to test constraints and aid testing.
+	define( 'CRON_CONTROL_CACHE_BUCKET_SIZE', 0 );
+	define( 'CRON_CONTROL_MAX_CACHE_BUCKETS', PHP_INT_MAX / 2 );
+
 	require dirname( dirname( __FILE__ ) ) . '/cron-control.php';
 
 	// Plugin loads after `wp_install()` is called, so we compensate.

tests/tests/class-events-store-tests.php

@@ -144,4 +144,31 @@ class Events_Store_Tests extends \WP_UnitTestCase {
 
 		$this->assertEmpty( Utils::get_events_from_store() );
 	}
+
+	/**
+	 * Test event-cache splitting
+	 */
+	function test_excessive_event_creation() {
+		$timestamp_base = time() + ( 1 * \HOUR_IN_SECONDS );
+
+		$dummy_text = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam enim ante, maximus nec nisi ut, finibus ultrices orci. Maecenas suscipit, est eu suscipit sagittis, enim massa dignissim augue, sagittis gravida dolor nulla ut mi. Phasellus venenatis bibendum cursus. Aliquam a erat purus. Nulla elit nunc, egestas eget eros iaculis, interdum tincidunt elit. Vivamus vel blandit nisl. Proin in ornare dolor, convallis porta sem. Mauris rutrum nibh et ornare egestas. Mauris ultricies diam at nunc tristique rutrum. Aliquam varius non leo vel luctus. Vestibulum sagittis scelerisque ante, non faucibus nibh accumsan sed.';
+		$args       = array_fill( 0, 15, $dummy_text );
+
+		for ( $i = 1; $i <= 100; $i++ ) {
+			$timestamp = $timestamp_base + $i;
+			$action    = 'excessive_test_event_' . $i;
+			wp_schedule_single_event( $timestamp, $action, $args );
+		}
+
+		get_option( 'cron' );
+
+		$cached = wp_cache_get( \Automattic\WP\Cron_Control\Events_Store::CACHE_KEY );
+
+		$this->assertArrayHasKey( 'incrementer', $cached );
+		$this->assertArrayHasKey( 'buckets',     $cached );
+		$this->assertArrayHasKey( 'event_count', $cached );
+
+		$this->assertEquals( 4,   $cached['buckets'] );
+		$this->assertEquals( 100, $cached['event_count'] );
+	}
 }

tests/tests/class-misc-tests.php

@@ -32,13 +32,21 @@ class Misc_Tests extends \WP_UnitTestCase {
 	}
 
 	/**
-	 * Expected values for certain constants
+	 * Expected values for certain Core constants
 	 */
-	function test_constants() {
+	function test_core_constants() {
 		$this->assertTrue( defined( 'DISABLE_WP_CRON' ) );
 		$this->assertTrue( defined( 'ALTERNATE_WP_CRON' ) );
 
 		$this->assertTrue( constant( 'DISABLE_WP_CRON' ) );
 		$this->assertFalse( constant( 'ALTERNATE_WP_CRON' ) );
 	}
+
+	/**
+	 * Confirm that constants are properly constrained
+	 */
+	function test_event_cache_constants() {
+		$this->assertEquals( 256 * \KB_IN_BYTES, \Automattic\WP\Cron_Control\CACHE_BUCKET_SIZE );
+		$this->assertEquals( 250,                \Automattic\WP\Cron_Control\MAX_CACHE_BUCKETS );
+	}
 }

tests/tests/class-utils-tests.php

+<?php
+/**
+ * Test plugin's utility functions
+ *
+ * @package a8c_Cron_Control
+ */
+
+namespace Automattic\WP\Cron_Control\Tests;
+use Automattic\WP\Cron_Control;
+
+/**
+ * Class Utils_Tests
+ */
+class Utils_Tests extends \WP_UnitTestCase {
+	/**
+	 * Prepare test environment
+	 */
+	function setUp() {
+		parent::setUp();
+
+		// make sure the schedule is clear.
+		_set_cron_array( array() );
+	}
+
+	/**
+	 * Clean up after our tests
+	 */
+	function tearDown() {
+		// make sure the schedule is clear.
+		_set_cron_array( array() );
+
+		parent::tearDown();
+	}
+
+	/**
+	 * Test functions that manipulate Core's cron format
+	 */
+	function test_events_array_collapse_and_inflate() {
+		$event_one = array(
+			'timestamp' => strtotime( '+15 minutes' ),
+			'action'    => 'test_event_1',
+			'args'      => array(
+				'test' => true,
+			),
+		);
+		wp_schedule_single_event( $event_one['timestamp'], $event_one['action'], $event_one['args'] );
+
+		$event_two = array(
+			'timestamp' => $event_one['timestamp'],
+			'action'    => 'test_event_2',
+			'args'      => array(
+				'fake_event' => 1,
+				'fake_args'  => array(
+					'thing' => 12,
+				),
+			),
+		);
+		wp_schedule_single_event( $event_two['timestamp'], $event_two['action'], $event_two['args'] );
+
+		$event_three = array(
+			'timestamp' => strtotime( '+30 minutes' ),
+			'action'    => 'test_event_3',
+			'args'      => array(
+				'post_id' => 1234,
+			),
+		);
+		wp_schedule_single_event( $event_three['timestamp'], $event_three['action'], $event_three['args'] );
+
+		$cron = get_option( 'cron' );
+
+		$collapsed = Cron_Control\collapse_events_array( $cron );
+
+		$this->assertEquals( 3, count( $collapsed ) );
+
+		$inflated = Cron_Control\inflate_collapsed_events_array( $collapsed );
+
+		$this->assertEquals( $cron, $inflated );
+
+		_set_cron_array( $inflated );
+
+		$this->assertEquals( $event_one['timestamp'], wp_next_scheduled( $event_one['action'], $event_one['args'] ) );
+		$this->assertEquals( $event_two['timestamp'], wp_next_scheduled( $event_two['action'], $event_two['args'] ) );
+		$this->assertEquals( $event_three['timestamp'], wp_next_scheduled( $event_three['action'], $event_three['args'] ) );
+	}
+}