Commit 86f8471e authored by Erick Hitter's avatar Erick Hitter Committed by GitHub
Browse files

Merge pull request #114 from Automattic/add/cli-orchestration

Support running events via CLI
parents 570b3ed7 0d50a3ad
......@@ -12,6 +12,8 @@ class Events extends Singleton {
*/
const LOCK = 'run-events';
const DISABLE_RUN_OPTION = 'a8c_cron_control_disable_run';
private $concurrent_action_whitelist = array();
private $running_event = null;
......@@ -45,7 +47,7 @@ class Events extends Singleton {
}
// Flag is used in many contexts, so should be set for all of our requests, regardless of the action
define( 'DOING_CRON', true );
set_doing_cron();
// When running events, allow for long-running ones, and non-blocking trigger requests
if ( REST_API::ENDPOINT_RUN === $endpoint ) {
......@@ -120,10 +122,10 @@ class Events extends Singleton {
$current_events = $this->reduce_queue( $current_events );
}
// Combine with Internal Events and return necessary data to process the event queue
// Combine with Internal Events
// TODO: un-nest array, which is nested for legacy reasons
return array(
'events' => array_merge( $current_events, $internal_events ),
'endpoint' => get_rest_url( null, REST_API::API_NAMESPACE . '/' . REST_API::ENDPOINT_RUN ),
);
}
......@@ -466,6 +468,46 @@ class Events extends Singleton {
$this->do_lock_cleanup( $this->running_event );
}
/**
* Return status of automatic event execution
*
* @return int 0 if run is enabled, 1 if run is disabled indefinitely, otherwise timestamp when execution will resume
*/
public function run_disabled() {
$disabled = (int) get_option( self::DISABLE_RUN_OPTION, 0 );
if ( $disabled <= 1 || $disabled > time() ) {
return $disabled;
}
$this->update_run_status( 0 );
return 0;
}
/**
* Set automatic execution status
*
* 0 if run is enabled, 1 if run is disabled indefinitely, otherwise timestamp when execution will resume
*
* @param int $new_status
* @return bool
*/
public function update_run_status( $new_status ) {
$new_status = absint( $new_status );
// Don't store a past timestamp
if ( $new_status > 1 && $new_status < time() ) {
return false;
}
// Nothing to do, but `update_option()` will return false
if ( $new_status === $this->run_disabled() ) {
return false;
}
return update_option( self::DISABLE_RUN_OPTION, $new_status );
}
}
Events::instance();
......@@ -50,8 +50,17 @@ class REST_API extends Singleton {
* For monitoring and alerting, also provides the total number of pending events
*/
public function get_events() {
// Provides `events` and `endpoint` keys needed to run events
$response_array = Events::instance()->get_events();
$response_array = array(
'events' => array(),
'orchestrate_disabled' => Events::instance()->run_disabled(),
);
// Include events only when automatic execution is enabled
if ( 0 === $response_array['orchestrate_disabled'] ) {
$response_array = array_merge( $response_array, Events::instance()->get_events() );
}
$response_array['endpoint'] = get_rest_url( null, self::API_NAMESPACE . '/' . self::ENDPOINT_RUN );
// Provide pending event count for monitoring etc
$response_array['total_events_pending'] = count_events_by_status( Events_Store::STATUS_PENDING );
......@@ -63,6 +72,18 @@ class REST_API extends Singleton {
* Execute a specific event
*/
public function run_event( $request ) {
// Stop if event execution is blocked
$run_disabled = Events::instance()->run_disabled();
if ( 0 !== $run_disabled ) {
if ( 1 === $run_disabled ) {
$message = __( 'Automatic event execution is disabled indefinitely.', 'automattic-cron-control' );
} else {
$message = sprintf( __( 'Automatic event execution is disabled until %s (%d).', 'automattic-cron-control' ), date( 'Y-m-d H:i:s T', $run_disabled ), $run_disabled );
}
return rest_ensure_response( new \WP_Error( 'automatic-execution-disabled', $message, array( 'status' => 403, ) ) );
}
// Parse request for details needed to identify the event to execute
// `$timestamp` is, unsurprisingly, the Unix timestamp the event is scheduled for
// `$action` is the md5 hash of the action used when the event is registered
......
......@@ -115,3 +115,16 @@ function parse_request() {
return $parsed_request;
}
/**
* Consistently set flag Core uses to indicate cron execution is ongoing
*/
function set_doing_cron() {
if ( ! defined( 'DOING_CRON' ) ) {
define( 'DOING_CRON', true );
}
// WP 4.8 introduced the `wp_doing_cron()` function and filter
// These can be used to override the `DOING_CRON` constant, which may cause problems for plugin's requests
add_filter( 'wp_doing_cron', '__return_true', 99999 );
}
......@@ -9,23 +9,30 @@ if ( ! defined( '\WP_CLI' ) || ! \WP_CLI ) {
/**
* Prepare environment
*/
if ( ! \Automattic\WP\Cron_Control\Events_Store::is_installed() ) {
function prepare_environment() {
// Only interfere with `cron-control` commands
$cmd = \WP_CLI::get_runner()->arguments;
if ( ! is_array( $cmd ) || ! isset( $cmd['0'] ) ) {
return;
}
$cmd = $cmd[0];
if ( false === strpos( $cmd, 'cron-control' ) ) {
if ( false === strpos( $cmd[0], 'cron-control' ) ) {
return;
}
// Create table and die, to ensure command runs with proper state
if ( ! \Automattic\WP\Cron_Control\Events_Store::is_installed() ) {
\Automattic\WP\Cron_Control\Events_Store::instance()->cli_create_tables();
\WP_CLI::error( __( 'Cron Control installation completed. Please try again.', 'automattic-cron-control' ) );
}
// Set DOING_CRON when appropriate
if ( isset( $cmd[1] ) && 'orchestrate' === $cmd[1] ) {
\Automattic\WP\Cron_Control\set_doing_cron();
}
}
prepare_environment();
/**
* Consistent time format across commands
......@@ -60,4 +67,6 @@ require __DIR__ . '/wp-cli/class-cache.php';
require __DIR__ . '/wp-cli/class-events.php';
require __DIR__ . '/wp-cli/class-lock.php';
require __DIR__ . '/wp-cli/class-one-time-fixers.php';
require __DIR__ . '/wp-cli/class-orchestrate.php';
require __DIR__ . '/wp-cli/class-orchestrate-runner.php';
require __DIR__ . '/wp-cli/class-rest-api.php';
......@@ -128,9 +128,7 @@ class Events extends \WP_CLI_Command {
\WP_CLI::confirm( sprintf( __( 'Run this event?', 'automattic-cron-control' ) ) );
// Environment preparation
if ( ! defined( 'DOING_CRON' ) ) {
define( 'DOING_CRON', true );
}
\Automattic\WP\Cron_Control\set_doing_cron();
// Run the event
$run = \Automattic\WP\Cron_Control\run_event( $event->timestamp, $event->action_hashed, $event->instance, true );
......
<?php
namespace Automattic\WP\Cron_Control\CLI;
/**
* Commands used by the Go-based runner to execute events
*/
class Orchestrate_Runner extends \WP_CLI_Command {
/**
* List the next set of events to run; meant for Runner
*
* Will not be all events, just those atop the curated queue
*
* Not intended for human use, rather it powers the Go-based Runner. Use the `events list` command instead.
*
* @subcommand list-due-batch
*/
public function list_due_now( $args, $assoc_args ) {
if ( 0 !== \Automattic\WP\Cron_Control\Events::instance()->run_disabled() ) {
\WP_CLI::error( __( 'Automatic event execution is disabled', 'automattic-cron-control' ) );
}
$events = \Automattic\WP\Cron_Control\Events::instance()->get_events();
$format = \WP_CLI\Utils\get_flag_value( $assoc_args, 'format', 'table' );
\WP_CLI\Utils\format_items( $format, $events['events'], array(
'timestamp',
'action',
'instance',
) );
}
/**
* Run a given event; meant for Runner
*
* Not intended for human use, rather it powers the Go-based Runner. Use the `events run` command instead.
*
* @subcommand run
* @synopsis --timestamp=<timestamp> --action=<action-hashed> --instance=<instance>
*/
public function run_event( $args, $assoc_args ) {
if ( 0 !== \Automattic\WP\Cron_Control\Events::instance()->run_disabled() ) {
\WP_CLI::error( __( 'Automatic event execution is disabled', 'automattic-cron-control' ) );
}
$timestamp = \WP_CLI\Utils\get_flag_value( $assoc_args, 'timestamp', null );
$action = \WP_CLI\Utils\get_flag_value( $assoc_args, 'action', null );
$instance = \WP_CLI\Utils\get_flag_value( $assoc_args, 'instance', null );
if ( ! is_numeric( $timestamp ) ) {
\WP_CLI::error( __( 'Invalid timestamp', 'automattic-cron-control' ) );
}
if ( ! is_string( $action ) ) {
\WP_CLI::error( __( 'Invalid action', 'automattic-cron-control' ) );
}
if( ! is_string( $instance ) ) {
\WP_CLI::error( __( 'Invalid instance', 'automattic-cron-control' ) );
}
$now = time();
if ( $timestamp > $now ) {
\WP_CLI::error( sprintf( __( 'Given timestamp is for %1$s GMT, %2$s from now. The event\'s existence was not confirmed, and no attempt was made to execute it.', 'automattic-cron-control' ), date_i18n( TIME_FORMAT, $timestamp ), human_time_diff( $now, $timestamp ) ) );
}
// Prepare environment
\Automattic\WP\Cron_Control\set_doing_cron();
// Run the event
$run = \Automattic\WP\Cron_Control\run_event( $timestamp, $action, $instance );
if ( is_wp_error( $run ) ) {
\WP_CLI::error( $run->get_error_message() );
} elseif ( isset( $run['success'] ) && true === $run['success'] ) {
\WP_CLI::success( $run['message'] );
} else {
\WP_CLI::error( $run['message'] );
}
}
/**
* Get some details needed to execute events; meant for Runner
*
* Not intended for human use, rather it powers the Go-based Runner. Use the `orchestrate manage-automatic-execution` command instead.
*
* @subcommand get-info
*/
public function get_info( $args, $assoc_args ) {
$info = array(
array(
'multisite' => is_multisite() ? 1 : 0,
'siteurl' => site_url(),
'disabled' => \Automattic\WP\Cron_Control\Events::instance()->run_disabled(),
),
);
$format = \WP_CLI\Utils\get_flag_value( $assoc_args, 'format', 'table' );
\WP_CLI\Utils\format_items( $format, $info, array_keys( $info[0] ) );
}
}
\WP_CLI::add_command( 'cron-control orchestrate runner-only', 'Automattic\WP\Cron_Control\CLI\Orchestrate_Runner' );
<?php
namespace Automattic\WP\Cron_Control\CLI;
/**
* Commands to manage automatic event execution
*/
class Orchestrate extends \WP_CLI_Command {
/**
* Check the status of automatic event execution
*
* @subcommand check-status
*/
public function get_automatic_execution_status( $args, $assoc_args ) {
$status = \Automattic\WP\Cron_Control\Events::instance()->run_disabled();
switch ( $status ) {
case 0 :
$status = __( 'Automatic execution is enabled', 'automattic-cron-control' );
break;
case 1 :
$status = __( 'Automatic execution is disabled indefinitely', 'automattic-cron-control' );
break;
default :
$status = sprintf( __( 'Automatic execution is disabled until %s', 'automattic-cron-control' ), date_i18n( 'Y-m-d H:i:s T', $status ) );
break;
}
\WP_CLI::log( $status );
}
/**
* Change status of automatic event execution
*
* When using the Go-based runner, it may be necessary to stop execution for a period, or indefinitely
*
* @subcommand manage-automatic-execution
* @synopsis [--enable] [--disable] [--disable_until=<disable_until>]
*/
public function manage_automatic_execution( $args, $assoc_args ) {
// Update execution status
$disable_ts = \WP_CLI\Utils\get_flag_value( $assoc_args, 'disable_until', 0 );
$disable_ts = absint( $disable_ts );
if ( \WP_CLI\Utils\get_flag_value( $assoc_args, 'enable', false ) ) {
$updated = \Automattic\WP\Cron_Control\Events::instance()->update_run_status( 0 );
if ( $updated ) {
\WP_CLI::success( __( 'Enabled', 'automattic-cron-control' ) );
return;
}
\WP_CLI::error( __( 'Could not enable automatic execution. Please check the current status.', 'automattic-cron-control' ) );
} elseif ( \WP_CLI\Utils\get_flag_value( $assoc_args, 'disable', false ) ) {
$updated = \Automattic\WP\Cron_Control\Events::instance()->update_run_status( 1 );
if ( $updated ) {
\WP_CLI::success( __( 'Disabled', 'automattic-cron-control' ) );
return;
}
\WP_CLI::error( __( 'Could not disable automatic execution. Please check the current status.', 'automattic-cron-control' ) );
} elseif( $disable_ts > 0 ) {
if ( $disable_ts > time() ) {
$updated = \Automattic\WP\Cron_Control\Events::instance()->update_run_status( $disable_ts );
if ( $updated ) {
\WP_CLI::success( sprintf( __( 'Disabled until %s', 'automattic-cron-control' ), date_i18n( 'Y-m-d H:i:s T', $disable_ts ) ) );
return;
}
\WP_CLI::error( __( 'Could not disable automatic execution. Please check the current status.', 'automattic-cron-control' ) );
} else {
\WP_CLI::error( __( 'Timestamp is in the past.', 'automattic-cron-control' ) );
}
}
\WP_CLI::error( __( 'Please provide a valid action.', 'automattic-cron-control' ) );
}
}
\WP_CLI::add_command( 'cron-control orchestrate', 'Automattic\WP\Cron_Control\CLI\Orchestrate' );
# MakeFile for cron-control-runner
DIR=`pwd`
GO=/usr/local/go/bin/go
EXECUTABLE=cron-control-runner
all:
GOPATH=${DIR}/../ ${GO} build -o ${DIR}/../bin/${EXECUTABLE}
clean:
@ rm -f ${DIR}/../bin/${EXECUTABLE}
Cron Control Go Runner
======================
In addition to the REST API endpoints that can be used to run events, a Go-based runner is provided.
# Installation
1. Build the binary as described below.
2. Copy `init.sh` to `/etc/init/cron-control-runner`
3. To override default configuration, copy `defaults` to `/etc/default/cron-control-runner` and modify as needed
4. Run `update-rc.d cron-control-runner defaults`
5. Start the runner: `/etc/init.d/cron-control-runner start`
6. Check the runner's status: `/etc/init.d/cron-control-runner status`
# Runner options
* `-cli` string
* Path to WP-CLI binary (default `/usr/local/bin/wp`)
* `-heartbeat` int
* Heartbeat interval in seconds (default `60`)
* `-log` string
* Log path, omit to log to Stdout (default `os.Stdout`)
* `-network` int
* WordPress network ID, `0` to disable (default `0`)
* `-workers-get` int
* Number of workers to retrieve events (default `1`)
* Increase for multisite instances so that sites are retrieved in a timely manner
* `-workers-run` int
* Number of workers to run events (default `5`)
* Increase for cron-heavy sites and multisite instances so that events are run in a timely manner
* `-wp` string
* Path to WordPress installation (default `/var/www/html`)
# Build the binary
If building on the target system, or under the same OS as the target machine, simply:
```
make
```
If building from a different OS:
```
env GOOS=linux make
```
Substitute `linux` with your target OS.
# DAEMON_ARGS="-workers-run 10"
#!/bin/sh
### BEGIN INIT INFO
# Provides: cron-control-runner
# Required-Start: $network $local_fs
# Required-Stop: $network $local_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Runner for Cron Control events
# Description: Runner for Cron Control events
### END INIT INFO
. /lib/lsb/init-functions
NAME=cron-control-runner
USER=www-data
DAEMON="/usr/local/bin/${NAME}"
DAEMON_ARGS=""
PIDFILE="/var/run/${NAME}.pid"
# Overrides
[ -f "/etc/default/$NAME" ] && . /etc/default/$NAME
# If the daemon is not there, then exit.
test -x $DAEMON || exit 5
case $1 in
start)
# Checked the PID file exists and check the actual status of process
if [ -e $PIDFILE ]; then
status_of_proc -p $PIDFILE $DAEMON "$NAME process" && status="0" || status="$?"
# If the status is SUCCESS then don't need to start again.
if [ $status = "0" ]; then
exit # Exit
fi
fi
# Start the daemon.
log_daemon_msg "Starting the process" "$NAME"
# Start the daemon with the help of start-stop-daemon
# Log the message appropriately
if start-stop-daemon --start --chuid $USER --background --oknodo --pidfile $PIDFILE --make-pidfile --exec $DAEMON -- $DAEMON_ARGS; then
log_end_msg 0
else
log_end_msg 1
fi
;;
stop)
# Stop the daemon.
if [ -e $PIDFILE ]; then
status_of_proc -p $PIDFILE $DAEMON "Stoppping the $NAME process" && status="0" || status="$?"
if [ "$status" = 0 ]; then
start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
/bin/rm -rf $PIDFILE
fi
else
log_daemon_msg "$NAME process is not running"
log_end_msg 0
fi
;;
restart)
# Restart the daemon.
$0 stop && sleep 2 && $0 start
;;
status)
# Check the status of the process.
if [ -e $PIDFILE ]; then
status_of_proc -p $PIDFILE $DAEMON "$NAME process" && exit 0 || exit $?
else
log_daemon_msg "$NAME Process is not running"
log_end_msg 0
fi
;;
*)
# For invalid arguments, print the usage message.
echo "Usage: $0 {start|stop|restart|status}"
exit 2
;;
esac
\ No newline at end of file
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"math/rand"
"os"
"os/exec"
"os/signal"
"path/filepath"
"sync/atomic"
"syscall"
"time"
)
type siteInfo struct {
Multisite int
Siteurl string
Disabled int
}
type site struct {
URL string
}
type event struct {
URL string
Timestamp int
Action string
Instance string
}
var (
wpCliPath string
wpNetwork int
wpPath string
numGetWorkers int
numRunWorkers int
getEventsInterval int
heartbeatInt int
disabledLoopCount uint64
eventRunErrCount uint64
eventRunSuccessCount uint64
logger *log.Logger
logDest string
debug bool
)
const getEventsBreak time.Duration = time.Second
const runEventsBreak time.Duration = time.Second * 10
func init() {
flag.StringVar(&wpCliPath, "cli", "/usr/local/bin/wp", "Path to WP-CLI binary")
flag.IntVar(&wpNetwork, "network", 0, "WordPress network ID, `0` to disable")
flag.StringVar(&wpPath, "wp", "/var/www/html", "Path to WordPress installation")
flag.IntVar(&numGetWorkers, "workers-get", 1, "Number of workers to retrieve events")
flag.IntVar(&numRunWorkers, "workers-run", 5, "Number of workers to run events")
flag.IntVar(&getEventsInterval, "get-events-interval", 60, "Seconds between event retrieval")
flag.IntVar(&heartbeatInt, "heartbeat", 60, "Heartbeat interval in seconds")
flag.StringVar(&logDest, "log", "os.Stdout", "Log path, omit to log to Stdout")
flag.BoolVar(&debug, "debug", false, "Include additional log data for debugging")
flag.Parse()
setUpLogger()
// TODO: Should check for wp-config.php instead?
validatePath(&wpCliPath, "WP-CLI path")
validatePath(&wpPath, "WordPress path")
}
func main() {
logger.Printf("Starting with %d event-retreival worker(s) and %d event worker(s)", numGetWorkers, numRunWorkers)
logger.Printf("Retrieving events every %d seconds", getEventsInterval)
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
sites := make(chan site)
events := make(chan event)
go spawnEventRetrievers(sites, events)
go spawnEventWorkers(events)
go retrieveSitesPeriodically(sites)
go heartbeat()
caughtSig := <-sig
logger.Printf("Stopping, got signal %s", caughtSig)
}
func spawnEventRetrievers(sites <-chan site, queue chan<- event) {
for w := 1; w <= numGetWorkers; w++ {
go queueSiteEvents(w, sites, queue)
}
}