From 78bbc2ffa53962876b9458227a204e65f1df4f70 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Wed, 9 Oct 2024 22:59:08 +0900 Subject: [PATCH 01/16] Initial structure of Queue system, with Redis driver --- common/framework/Queue.php | 305 ++++++++++++++++++++ common/framework/drivers/QueueInterface.php | 63 ++++ common/framework/drivers/queue/base.php | 0 common/framework/drivers/queue/redis.php | 167 +++++++++++ common/scripts/cron.php | 34 +++ 5 files changed, 569 insertions(+) create mode 100644 common/framework/Queue.php create mode 100644 common/framework/drivers/QueueInterface.php create mode 100644 common/framework/drivers/queue/base.php create mode 100644 common/framework/drivers/queue/redis.php create mode 100644 common/scripts/cron.php diff --git a/common/framework/Queue.php b/common/framework/Queue.php new file mode 100644 index 000000000..810d4513f --- /dev/null +++ b/common/framework/Queue.php @@ -0,0 +1,305 @@ + $class_name::getName(), + 'required' => $class_name::getRequiredConfig(), + 'optional' => $class_name::getOptionalConfig(), + ]; + } + } + foreach (self::$_drivers as $driver_name => $driver) + { + if ($driver->isSupported()) + { + $result[$driver_name] = [ + 'name' => $driver->getName(), + 'required' => $driver->getRequiredConfig(), + 'optional' => $driver->getOptionalConfig(), + ]; + } + } + ksort($result); + return $result; + } + + /** + * Add a task. + * + * The handler can be in one of the following formats: + * - Global function, e.g. myHandler + * - ClassName::staticMethodName + * - ClassName::getInstance()->methodName + * - new ClassName()->methodName + * + * Once identified and/or instantiated, the handler will be passed $args + * and $options, in that order. Each of them must be a single object. + * + * It is strongly recommended that you write a dedicated method to handle + * queued tasks, rather than reusing an existing method with a potentially + * incompatible structure. If you must to call an existing method, + * you should consider writing a wrapper. + * + * Any value returned by the handler will be discarded. If you throw an + * exception, it may be logged, but it will not cause a fatal error. + * + * @param string $handler + * @param ?object $args + * @param ?object $options + * @return int + */ + public static function addTask(string $handler, ?object $args = null, ?object $options = null): int + { + $driver_name = config('queue.driver'); + if (!$driver_name) + { + throw new Exceptions\FeatureDisabled('Queue not configured'); + } + + $driver = self::getDriver($driver_name); + if (!$driver) + { + throw new Exceptions\FeatureDisabled('Queue not configured'); + } + + return $driver->addTask($handler, $args, $options); + } + + /** + * Get the first task to execute immediately. + * + * If no tasks are pending, this method will return null. + * Detailed scheduling of tasks will be handled by each driver. + * + * @param int $blocking + * @return ?object + */ + public static function getTask(int $blocking = 0): ?object + { + $driver_name = config('queue.driver'); + if (!$driver_name) + { + throw new Exceptions\FeatureDisabled('Queue not configured'); + } + + $driver = self::getDriver($driver_name); + if (!$driver) + { + throw new Exceptions\FeatureDisabled('Queue not configured'); + } + + return $driver->getTask($blocking); + } + + /** + * Check the process key. + * + * @param string $key + * @return bool + */ + public static function checkKey(string $key): bool + { + return config('queue.key') === $key; + } + + /** + * Process the queue. + * + * This will usually be called by a separate script, run every minute + * through an external scheduler such as crontab or systemd. + * + * If you are on a shared hosting service, you may also call a URL + * using a "web cron" service provider. + * + * @param int $timeout + * @return void + */ + public static function process(int $timeout): void + { + // Increase the time limit. This may or may not work. + set_time_limit(min(60, $timeout)); + + // This part will run in a loop until timeout. + $process_start_time = microtime(true); + while (true) + { + // Get a task from the driver. + $loop_start_time = microtime(true); + $task = self::getTask(1); + + // Wait 1 second and loop back. + if ($task) + { + // Find the handler for the task. + $task->handler = trim($task->handler, '\\()'); + $handler = null; + try + { + if (preg_match('/^(?:\\\\)?([\\\\\\w]+)::(\\w+)$/', $task->handler, $matches)) + { + $class_name = '\\' . $matches[1]; + $method_name = $matches[2]; + if (class_exists($class_name) && method_exists($class_name, $method_name)) + { + $handler = [$class_name, $method_name]; + } + else + { + error_log('RxQueue: task handler not found: ' . $task->handler); + } + } + elseif (preg_match('/^(?:\\\\)?([\\\\\\w]+)::(\\w+)(?:\(\))?->(\\w+)$/', $task->handler, $matches)) + { + $class_name = '\\' . $matches[1]; + $initializer_name = $matches[2]; + $method_name = $matches[3]; + if (class_exists($class_name) && method_exists($class_name, $initializer_name)) + { + $obj = $class_name::$initializer_name(); + if (method_exists($obj, $method_name)) + { + $handler = [$obj, $method_name]; + } + else + { + error_log('RxQueue: task handler not found: ' . $task->handler); + } + } + else + { + error_log('RxQueue: task handler not found: ' . $task->handler); + } + } + elseif (preg_match('/^new (?:\\\\)?([\\\\\\w]+)(?:\(\))?->(\\w+)$/', $task->handler, $matches)) + { + $class_name = '\\' . $matches[1]; + $method_name = $matches[2]; + if (class_exists($class_name) && method_exists($class_name, $method_name)) + { + $obj = new $class_name(); + $handler = [$obj, $method_name]; + } + else + { + error_log('RxQueue: task handler not found: ' . $task->handler); + } + } + else + { + if (function_exists('\\' . $task->handler)) + { + $handler = '\\' . $task->handler; + } + else + { + error_log('RxQueue: task handler not found: ' . $task->handler); + } + } + } + catch (\Throwable $th) + { + error_log(vsprintf('RxQueue: task handler %s could not be accessed: %s in %s:%d', [ + $task->handler, + get_class($th), + $th->getFile(), + $th->getLine(), + ])); + } + + // Call the handler. + try + { + if ($handler) + { + call_user_func($handler, $task->args, $task->options); + } + } + catch (\Throwable $th) + { + error_log(vsprintf('RxQueue: task handler %s threw %s in %s:%d', [ + $task->handler, + get_class($th), + $th->getFile(), + $th->getLine(), + ])); + } + } + + // If the timeout is imminent, break the loop. + $process_elapsed_time = microtime(true) - $process_start_time; + if ($process_elapsed_time > $timeout - 2) + { + break; + } + + // If there was no task, wait 1 second to make sure that the loop isn't too tight. + $loop_elapsed_time = microtime(true) - $loop_start_time; + if (!$task && $loop_elapsed_time < 1) + { + usleep(intval((1 - $loop_elapsed_time) * 1000000)); + } + } + } +} diff --git a/common/framework/drivers/QueueInterface.php b/common/framework/drivers/QueueInterface.php new file mode 100644 index 000000000..f3c5697f5 --- /dev/null +++ b/common/framework/drivers/QueueInterface.php @@ -0,0 +1,63 @@ +_conn = new \Redis; + $this->_conn->connect($config['host'], $config['port'] ?? 6379); + if(isset($config['user']) || isset($config['pass'])) + { + $auth = []; + if (isset($config['user']) && $config['user']) $auth[] = $config['user']; + if (isset($config['pass']) && $config['pass']) $auth[] = $config['pass']; + $this->_conn->auth(count($auth) > 1 ? $auth : $auth[0]); + } + if(isset($config['dbnum'])) + { + $this->_conn->select(intval($config['dbnum'])); + } + + $this->_key = 'RXQUEUE_' . substr(sha1(\RX_BASEDIR), 0, 16); + } + catch (\RedisException $e) + { + $this->_conn = null; + } + } + + /** + * Add a task. + * + * @param string $handler + * @param ?object $args + * @param ?object $options + * @return int + */ + public function addTask(string $handler, ?object $args = null, ?object $options = null): int + { + $value = serialize((object)[ + 'handler' => $handler, + 'args' => $args, + 'options' => $options, + ]); + + if ($this->_conn) + { + $result = $this->_conn->rPush($this->_key, $value); + return intval($result); + } + else + { + return 0; + } + } + + /** + * Get the first task. + * + * @param int $blocking + * @return ?object + */ + public function getTask(int $blocking = 0): ?object + { + if ($this->_conn) + { + if ($blocking > 0) + { + $result = $this->_conn->blpop($this->_key, $blocking); + if (is_array($result) && isset($result[1])) + { + return unserialize($result[1]); + } + else + { + return null; + } + } + else + { + $result = $this->_conn->lpop($this->_key); + if ($result) + { + return unserialize($result); + } + else + { + return null; + } + } + } + else + { + return null; + } + } +} diff --git a/common/scripts/cron.php b/common/scripts/cron.php new file mode 100644 index 000000000..ece9ee93f --- /dev/null +++ b/common/scripts/cron.php @@ -0,0 +1,34 @@ + Date: Wed, 9 Oct 2024 23:12:55 +0900 Subject: [PATCH 02/16] Add DB queue driver --- common/framework/drivers/queue/base.php | 0 common/framework/drivers/queue/db.php | 123 ++++++++++++++++++++++++ modules/module/schemas/task_queue.xml | 8 ++ 3 files changed, 131 insertions(+) delete mode 100644 common/framework/drivers/queue/base.php create mode 100644 common/framework/drivers/queue/db.php create mode 100644 modules/module/schemas/task_queue.xml diff --git a/common/framework/drivers/queue/base.php b/common/framework/drivers/queue/base.php deleted file mode 100644 index e69de29bb..000000000 diff --git a/common/framework/drivers/queue/db.php b/common/framework/drivers/queue/db.php new file mode 100644 index 000000000..90225f02d --- /dev/null +++ b/common/framework/drivers/queue/db.php @@ -0,0 +1,123 @@ +prepare('INSERT INTO task_queue (handler, args, options) VALUES (?, ?, ?)'); + $result = $stmt->execute([$handler, serialize($args), serialize($options)]); + return $result ? $oDB->getInsertID() : false; + } + + /** + * Get the first task. + * + * @param int $blocking + * @return ?object + */ + public function getTask(int $blocking = 0): ?object + { + $oDB = RFDB::getInstance(); + $stmt = $oDB->query('SELECT * FROM task_queue ORDER BY id LIMIT 1'); + $result = $stmt->fetchObject(); + $stmt->closeCursor(); + + if ($result) + { + $stmt = $oDB->prepare('DELETE FROM task_queue WHERE id = ?'); + $stmt->execute([$result->id]); + + $result->args = unserialize($result->args); + $result->options = unserialize($result->options); + return $result; + } + else + { + return null; + } + } +} diff --git a/modules/module/schemas/task_queue.xml b/modules/module/schemas/task_queue.xml new file mode 100644 index 000000000..6e991a700 --- /dev/null +++ b/modules/module/schemas/task_queue.xml @@ -0,0 +1,8 @@ + + + + + + + +
From d8370ff59b64ac67dcfec2041023ecaeca8ee5d8 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 00:07:35 +0900 Subject: [PATCH 03/16] Add dummy driver, clean up loose ends and start writing admin page --- common/framework/drivers/queue/db.php | 8 +- common/framework/drivers/queue/dummy.php | 96 ++++++++++++++++++++++++ modules/admin/conf/module.xml | 2 + modules/admin/lang/en.php | 1 + modules/admin/lang/ko.php | 1 + modules/admin/tpl/config_header.html | 1 + 6 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 common/framework/drivers/queue/dummy.php diff --git a/common/framework/drivers/queue/db.php b/common/framework/drivers/queue/db.php index 90225f02d..f92e5614f 100644 --- a/common/framework/drivers/queue/db.php +++ b/common/framework/drivers/queue/db.php @@ -10,12 +10,6 @@ use Rhymix\Framework\Drivers\QueueInterface; */ class DB implements QueueInterface { - /** - * The Redis connection is stored here. - */ - protected $_conn; - protected $_key; - /** * Create a new instance of the current Queue driver, using the given settings. * @@ -90,7 +84,7 @@ class DB implements QueueInterface $oDB = RFDB::getInstance(); $stmt = $oDB->prepare('INSERT INTO task_queue (handler, args, options) VALUES (?, ?, ?)'); $result = $stmt->execute([$handler, serialize($args), serialize($options)]); - return $result ? $oDB->getInsertID() : false; + return $result ? $oDB->getInsertID() : 0; } /** diff --git a/common/framework/drivers/queue/dummy.php b/common/framework/drivers/queue/dummy.php new file mode 100644 index 000000000..c39d10947 --- /dev/null +++ b/common/framework/drivers/queue/dummy.php @@ -0,0 +1,96 @@ + + + diff --git a/modules/admin/lang/en.php b/modules/admin/lang/en.php index f2c6eab8d..85dd256c1 100644 --- a/modules/admin/lang/en.php +++ b/modules/admin/lang/en.php @@ -8,6 +8,7 @@ $lang->subtitle_security = 'Security'; $lang->subtitle_advanced = 'Advanced'; $lang->subtitle_domains = 'Domains'; $lang->subtitle_debug = 'Debug'; +$lang->subtitle_queue = 'Async Queue'; $lang->subtitle_seo = 'SEO Settings'; $lang->subtitle_etc = 'Other Settings'; $lang->current_state = 'Current state'; diff --git a/modules/admin/lang/ko.php b/modules/admin/lang/ko.php index 0de6799a0..9b63d03f6 100644 --- a/modules/admin/lang/ko.php +++ b/modules/admin/lang/ko.php @@ -7,6 +7,7 @@ $lang->subtitle_notification = '알림 설정'; $lang->subtitle_security = '보안 설정'; $lang->subtitle_advanced = '고급 설정'; $lang->subtitle_debug = '디버그 설정'; +$lang->subtitle_queue = '비동기 작업'; $lang->subtitle_seo = 'SEO 설정'; $lang->subtitle_etc = '기타'; $lang->current_state = '현황'; diff --git a/modules/admin/tpl/config_header.html b/modules/admin/tpl/config_header.html index a8e7e7a1c..84e8b3a25 100644 --- a/modules/admin/tpl/config_header.html +++ b/modules/admin/tpl/config_header.html @@ -12,5 +12,6 @@
  • {$lang->subtitle_advanced}
  • {$lang->subtitle_debug}
  • {$lang->subtitle_seo}
    • +
  • {$lang->subtitle_queue}
  • {$lang->subtitle_sitelock}
    • From a2a1f3bcc6a0040e6ef5bdcc98d6d1a0cc53e546 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 01:45:14 +0900 Subject: [PATCH 04/16] Implement admin config screen for Queue --- common/framework/drivers/queue/redis.php | 4 +- .../admin/controllers/systemconfig/Queue.php | 115 +++++++++++++++++ modules/admin/lang/en.php | 13 ++ modules/admin/lang/ko.php | 13 ++ modules/admin/tpl/config_queue.html | 119 ++++++++++++++++++ modules/admin/tpl/js/queue_config.js | 16 +++ 6 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 modules/admin/controllers/systemconfig/Queue.php create mode 100644 modules/admin/tpl/config_queue.html create mode 100644 modules/admin/tpl/js/queue_config.js diff --git a/common/framework/drivers/queue/redis.php b/common/framework/drivers/queue/redis.php index 92cf7b034..c5d0c17bc 100644 --- a/common/framework/drivers/queue/redis.php +++ b/common/framework/drivers/queue/redis.php @@ -53,7 +53,7 @@ class Redis implements QueueInterface */ public static function getOptionalConfig(): array { - return ['user', 'pass', 'dbnum']; + return ['dbnum', 'user', 'pass']; } /** @@ -89,7 +89,7 @@ class Redis implements QueueInterface $this->_conn->select(intval($config['dbnum'])); } - $this->_key = 'RXQUEUE_' . substr(sha1(\RX_BASEDIR), 0, 16); + $this->_key = 'rxQueue_' . substr(hash_hmac('sha1', 'rxQueue_', config('crypto.authentication_key')), 0, 24); } catch (\RedisException $e) { diff --git a/modules/admin/controllers/systemconfig/Queue.php b/modules/admin/controllers/systemconfig/Queue.php new file mode 100644 index 000000000..d3f0c665c --- /dev/null +++ b/modules/admin/controllers/systemconfig/Queue.php @@ -0,0 +1,115 @@ + '127.0.0.1', + 'port' => '6379', + 'dbnum' => 0, + ]); + } + + $this->setTemplateFile('config_queue'); + } + + /** + * Update notification configuration. + */ + public function procAdminUpdateQueue() + { + $vars = Context::getRequestVars(); + + // Enabled? + $enabled = $vars->queue_enabled === 'Y'; + + // Validate the driver. + $drivers = RFQueue::getSupportedDrivers(); + $driver = trim($vars->queue_driver); + if (!array_key_exists($driver, $drivers)) + { + throw new Exception('1111'); + } + if ($enabled && (!$driver || $driver === 'dummy')) + { + throw new Exception('2222'); + } + + // Validate required and optional driver settings. + $driver_config = array(); + foreach ($drivers[$driver]['required'] as $conf_name) + { + $conf_value = trim($vars->{'queue_' . $driver . '_' . $conf_name} ?? ''); + if ($conf_value === '') + { + throw new Exception('3333'); + } + $driver_config[$conf_name] = $conf_value === '' ? null : $conf_value; + } + foreach ($drivers[$driver]['optional'] as $conf_name) + { + $conf_value = trim($vars->{'queue_' . $driver . '_' . $conf_name} ?? ''); + $driver_config[$conf_name] = $conf_value === '' ? null : $conf_value; + } + + // Validate the interval. + $interval = intval($vars->queue_interval ?? 1); + if ($interval < 1 || $interval > 10) + { + throw new Exception('4444'); + } + + // Validate the key. + $key = trim($vars->queue_key ?? ''); + if (strlen($key) < 16 || !ctype_alnum($key)) + { + throw new Exception('5555'); + } + + // Save system config. + Config::set("queue.enabled", $enabled); + Config::set("queue.driver", $driver); + Config::set("queue.interval", $interval); + Config::set("queue.key", $key); + Config::set("queue.$driver", $driver_config); + if (!Config::save()) + { + throw new Exception('msg_failed_to_save_config'); + } + + $this->setMessage('success_updated'); + $this->setRedirectUrl(Context::get('success_return_url') ?: getNotEncodedUrl('', 'module', 'admin', 'act', 'dispAdminConfigQueue')); + } +} diff --git a/modules/admin/lang/en.php b/modules/admin/lang/en.php index 85dd256c1..05f8bc4d0 100644 --- a/modules/admin/lang/en.php +++ b/modules/admin/lang/en.php @@ -283,6 +283,19 @@ $lang->og_extract_images_fallback = 'Use site default image only'; $lang->og_extract_hashtags = 'Extract Hashtags from Document'; $lang->og_use_nick_name = 'Include Author Name'; $lang->og_use_timestamps = 'Include Timestamps'; +$lang->cmd_queue_enabled = 'Use Task Queue'; +$lang->cmd_queue_enabled_help = 'The task queue will stop accepting new tasks if you uncheck the above.'; +$lang->cmd_queue_driver = 'Queue Driver'; +$lang->cmd_queue_driver_help = 'Select the driver for the task queue that suits your hosting environment and website needs.
    Some drivers such as Redis will need the corresponding program to be installed on the server.'; +$lang->cmd_queue_interval = 'Calling Interval'; +$lang->cmd_queue_interval_help = 'Use a scheduler such as crontab or systemd timer to call the script on a set interval.
    All tasks are processed as soon as possible regardless of the interval, but a short interval means quick recovery from any error.
    For web-based cron, this should not exceed the max_execution_time setting in php.ini.
    The max_execution_time on this server is %d seconds.'; +$lang->cmd_queue_call_script = 'Processing Script'; +$lang->cmd_queue_webcron_key = 'Webcron Auth Key'; +$lang->cmd_queue_config_keys['host'] = 'Host'; +$lang->cmd_queue_config_keys['port'] = 'Port'; +$lang->cmd_queue_config_keys['user'] = 'User'; +$lang->cmd_queue_config_keys['pass'] = 'Password'; +$lang->cmd_queue_config_keys['dbnum'] = 'DB Number'; $lang->autoinstall = 'EasyInstall'; $lang->last_week = 'Last Week'; $lang->this_week = 'This Week'; diff --git a/modules/admin/lang/ko.php b/modules/admin/lang/ko.php index 9b63d03f6..4e269e611 100644 --- a/modules/admin/lang/ko.php +++ b/modules/admin/lang/ko.php @@ -279,6 +279,19 @@ $lang->og_extract_images_fallback = '사이트 대표 이미지 사용'; $lang->og_extract_hashtags = '본문에서 해시태그 추출'; $lang->og_use_nick_name = '글 작성자 이름 표시'; $lang->og_use_timestamps = '글 작성/수정 시각 표시'; +$lang->cmd_queue_enabled = '비동기 작업 사용'; +$lang->cmd_queue_enabled_help = '체크를 해제하면 더이상 작업을 접수하지 않습니다.'; +$lang->cmd_queue_driver = '비동기 드라이버'; +$lang->cmd_queue_driver_help = '비동기 작업을 관리할 방법을 설정합니다. 호스팅 환경과 사이트의 필요에 맞추어 선택하세요.
    Redis 등 일부 드라이버는 서버에 해당 기능이 설치되어 있어야 사용할 수 있습니다.'; +$lang->cmd_queue_interval = '호출 간격'; +$lang->cmd_queue_interval_help = 'crontab, systemd timer, 웹크론 등을 사용하여 일정한 주기로 스크립트를 호출해 주십시오.
    모든 비동기 작업은 호출 간격과 무관하게 실시간으로 처리되나, 호출 간격이 짧으면 장애 발생시 신속하게 복구됩니다.
    웹크론 사용시에는 php.ini의 실행 시간 제한을 초과하지 않는 것이 좋습니다.
    이 서버의 max_execution_time은 %d초로 설정되어 있습니다.'; +$lang->cmd_queue_call_script = '작업 처리 스크립트'; +$lang->cmd_queue_webcron_key = '웹크론 인증키'; +$lang->cmd_queue_config_keys['host'] = '호스트'; +$lang->cmd_queue_config_keys['port'] = '포트'; +$lang->cmd_queue_config_keys['user'] = '아이디'; +$lang->cmd_queue_config_keys['pass'] = '암호'; +$lang->cmd_queue_config_keys['dbnum'] = 'DB 번호'; $lang->autoinstall = '쉬운 설치'; $lang->last_week = '지난주'; $lang->this_week = '이번주'; diff --git a/modules/admin/tpl/config_queue.html b/modules/admin/tpl/config_queue.html new file mode 100644 index 000000000..d6a3fc50e --- /dev/null +++ b/modules/admin/tpl/config_queue.html @@ -0,0 +1,119 @@ + + + + + +
    +

    {$XE_VALIDATOR_MESSAGE}

    +
    + + + +
    + + + + +
    + +

    {$lang->subtitle_queue}

    + +
    + +
    + +
    +

    {$lang->cmd_queue_enabled_help}

    +
    +
    + +
    + +
    + +

    {$lang->cmd_queue_driver_help}

    +
    +
    + + + + {@ $conf_names = array_merge($driver_definition['required'], $driver_definition['optional'])} + + + + {@ $conf_value = escape(config("queue.$driver_name.$conf_name"))} + {@ $text_keys = ['host', 'user']} + {@ $number_keys = ['port', 'dbnum']} + {@ $password_keys = ['pass']} + + +
    + +
    + +
    +
    + + + +
    + +
    + +
    +
    + + + +
    + +
    + +
    +
    + + + + + + +
    + +
    + +

    {$lang->cmd_queue_call_script}

    + +
    + +
    + +
    +
    + +
    + +
    + + {$lang->unit_min} +
    +

    {sprintf($lang->cmd_queue_interval_help, ini_get('max_execution_time'))|noescape}

    +
    +
    + +
    + +
    +
    + +
    +
    +
    diff --git a/modules/admin/tpl/js/queue_config.js b/modules/admin/tpl/js/queue_config.js new file mode 100644 index 000000000..53af72f2f --- /dev/null +++ b/modules/admin/tpl/js/queue_config.js @@ -0,0 +1,16 @@ +(function($) { + $(function() { + + $("#queue_driver").on("change", function() { + var selected_driver = $(this).val(); + $(this).parents("section").find("div.x_control-group.hidden-by-default, p.x_help-block.hidden-by-default").each(function() { + if ($(this).hasClass("show-for-" + selected_driver)) { + $(this).show(); + } else { + $(this).hide(); + } + }); + }).triggerHandler("change"); + + }); +})(jQuery); From d4cd5e3d756348888d81342feea887a1e8b612ea Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 09:43:41 +0900 Subject: [PATCH 05/16] Add notice and process count to Queue config screen --- modules/admin/controllers/systemconfig/Queue.php | 8 ++++++++ modules/admin/lang/en.php | 3 +++ modules/admin/lang/ko.php | 3 +++ modules/admin/tpl/config_queue.html | 12 ++++++++++++ 4 files changed, 26 insertions(+) diff --git a/modules/admin/controllers/systemconfig/Queue.php b/modules/admin/controllers/systemconfig/Queue.php index d3f0c665c..6ae53634c 100644 --- a/modules/admin/controllers/systemconfig/Queue.php +++ b/modules/admin/controllers/systemconfig/Queue.php @@ -91,6 +91,13 @@ class Queue extends Base throw new Exception('4444'); } + // Validate the process count. + $process_count = intval($vars->queue_process_count ?? 1); + if ($process_count < 1 || $process_count > 10) + { + throw new Exception('4444'); + } + // Validate the key. $key = trim($vars->queue_key ?? ''); if (strlen($key) < 16 || !ctype_alnum($key)) @@ -102,6 +109,7 @@ class Queue extends Base Config::set("queue.enabled", $enabled); Config::set("queue.driver", $driver); Config::set("queue.interval", $interval); + Config::set("queue.process_count", $process_count); Config::set("queue.key", $key); Config::set("queue.$driver", $driver_config); if (!Config::save()) diff --git a/modules/admin/lang/en.php b/modules/admin/lang/en.php index 05f8bc4d0..84efb9734 100644 --- a/modules/admin/lang/en.php +++ b/modules/admin/lang/en.php @@ -283,12 +283,15 @@ $lang->og_extract_images_fallback = 'Use site default image only'; $lang->og_extract_hashtags = 'Extract Hashtags from Document'; $lang->og_use_nick_name = 'Include Author Name'; $lang->og_use_timestamps = 'Include Timestamps'; +$lang->cmd_queue_description = 'Improve response times by processing time-consuming tasks, such as sending notifications, asynchronously.
    This is an experimental feature. It may not be stable depending on your hosting environment.'; $lang->cmd_queue_enabled = 'Use Task Queue'; $lang->cmd_queue_enabled_help = 'The task queue will stop accepting new tasks if you uncheck the above.'; $lang->cmd_queue_driver = 'Queue Driver'; $lang->cmd_queue_driver_help = 'Select the driver for the task queue that suits your hosting environment and website needs.
    Some drivers such as Redis will need the corresponding program to be installed on the server.'; $lang->cmd_queue_interval = 'Calling Interval'; $lang->cmd_queue_interval_help = 'Use a scheduler such as crontab or systemd timer to call the script on a set interval.
    All tasks are processed as soon as possible regardless of the interval, but a short interval means quick recovery from any error.
    For web-based cron, this should not exceed the max_execution_time setting in php.ini.
    The max_execution_time on this server is %d seconds.'; +$lang->cmd_queue_process_count = 'Process Count'; +$lang->cmd_queue_process_count_help = 'Use multiple processes to increase throughput. This may increase server load significantly.
    Keep a value of 1 unless you have a high-performance dedicated server.
    Multiprocessing is not supported when using web-based cron.'; $lang->cmd_queue_call_script = 'Processing Script'; $lang->cmd_queue_webcron_key = 'Webcron Auth Key'; $lang->cmd_queue_config_keys['host'] = 'Host'; diff --git a/modules/admin/lang/ko.php b/modules/admin/lang/ko.php index 4e269e611..e45e4ab20 100644 --- a/modules/admin/lang/ko.php +++ b/modules/admin/lang/ko.php @@ -279,12 +279,15 @@ $lang->og_extract_images_fallback = '사이트 대표 이미지 사용'; $lang->og_extract_hashtags = '본문에서 해시태그 추출'; $lang->og_use_nick_name = '글 작성자 이름 표시'; $lang->og_use_timestamps = '글 작성/수정 시각 표시'; +$lang->cmd_queue_description = '메일 발송, 푸시알림 등 시간이 오래 걸리거나 외부 서비스와 연동하는 작업을 비동기 처리하여 응답 속도를 개선합니다.
    실험적인 기능입니다. 호스팅 환경에 따라서는 안정적으로 작동하지 않을 수도 있습니다.'; $lang->cmd_queue_enabled = '비동기 작업 사용'; $lang->cmd_queue_enabled_help = '체크를 해제하면 더이상 작업을 접수하지 않습니다.'; $lang->cmd_queue_driver = '비동기 드라이버'; $lang->cmd_queue_driver_help = '비동기 작업을 관리할 방법을 설정합니다. 호스팅 환경과 사이트의 필요에 맞추어 선택하세요.
    Redis 등 일부 드라이버는 서버에 해당 기능이 설치되어 있어야 사용할 수 있습니다.'; $lang->cmd_queue_interval = '호출 간격'; $lang->cmd_queue_interval_help = 'crontab, systemd timer, 웹크론 등을 사용하여 일정한 주기로 스크립트를 호출해 주십시오.
    모든 비동기 작업은 호출 간격과 무관하게 실시간으로 처리되나, 호출 간격이 짧으면 장애 발생시 신속하게 복구됩니다.
    웹크론 사용시에는 php.ini의 실행 시간 제한을 초과하지 않는 것이 좋습니다.
    이 서버의 max_execution_time은 %d초로 설정되어 있습니다.'; +$lang->cmd_queue_process_count = '프로세스 갯수'; +$lang->cmd_queue_process_count_help = '여러 개의 프로세스를 동시에 실행하여 처리 용량을 늘립니다. 서버 부하가 증가할 수 있습니다.
    고성능 단독서버가 아니라면 1을 유지하시기 바랍니다.
    웹크론으로 호출한 경우에는 멀티프로세싱을 지원하지 않습니다.'; $lang->cmd_queue_call_script = '작업 처리 스크립트'; $lang->cmd_queue_webcron_key = '웹크론 인증키'; $lang->cmd_queue_config_keys['host'] = '호스트'; diff --git a/modules/admin/tpl/config_queue.html b/modules/admin/tpl/config_queue.html index d6a3fc50e..1e873d488 100644 --- a/modules/admin/tpl/config_queue.html +++ b/modules/admin/tpl/config_queue.html @@ -3,6 +3,10 @@ +
    +

    {$lang->cmd_queue_description}

    +
    +

    {$XE_VALIDATOR_MESSAGE}

    @@ -109,6 +113,14 @@ +
    + +
    + +

    {$lang->cmd_queue_process_count_help}

    +
    +
    +
    From 097cecece87f73055256eccfb0c20e7a7f168216 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 23:04:35 +0900 Subject: [PATCH 06/16] Add code samples to Queue config screen --- common/scripts/cron.php | 2 +- modules/admin/lang/en.php | 7 +++ modules/admin/lang/ko.php | 7 +++ modules/admin/tpl/config_queue.html | 64 +++++++++++++++++++++++++ modules/admin/tpl/css/queue_config.scss | 27 +++++++++++ modules/admin/tpl/js/queue_config.js | 19 +++++++- 6 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 modules/admin/tpl/css/queue_config.scss diff --git a/common/scripts/cron.php b/common/scripts/cron.php index ece9ee93f..96e57b384 100644 --- a/common/scripts/cron.php +++ b/common/scripts/cron.php @@ -30,5 +30,5 @@ else } // The rest of the work will be done by the Queue class. -$timeout = config('queue.interval') ?? 60; +$timeout = (config('queue.interval') ?? 1) * 60; Rhymix\Framework\Queue::process($timeout); diff --git a/modules/admin/lang/en.php b/modules/admin/lang/en.php index 84efb9734..6e9618e1e 100644 --- a/modules/admin/lang/en.php +++ b/modules/admin/lang/en.php @@ -299,6 +299,13 @@ $lang->cmd_queue_config_keys['port'] = 'Port'; $lang->cmd_queue_config_keys['user'] = 'User'; $lang->cmd_queue_config_keys['pass'] = 'Password'; $lang->cmd_queue_config_keys['dbnum'] = 'DB Number'; +$lang->msg_queue_instructions['same_as_php'] = '(same as PHP)'; +$lang->msg_queue_instructions['crontab1'] = 'Log into the server as the %s account and run crontab -e to paste the following content into your crontab. (DO NOT run it as root!)
    The %s directory in the example should be replaced with a path where logs can be recorded.'; +$lang->msg_queue_instructions['crontab2'] = 'If you change the calling interval below, the crontab interval must be adjusted accordingly.'; +$lang->msg_queue_instructions['webcron'] = 'Configure an external cron service to make a GET request to the following URL every minute, or following the interval set below.
    Check your logs to make sure that the cron service is reaching your website.'; +$lang->msg_queue_instructions['systemd1'] = 'Put the following content in /etc/systemd/system/rhymix-queue.service'; +$lang->msg_queue_instructions['systemd2'] = 'Put the following content in /etc/systemd/system/rhymix-queue.timer'; +$lang->msg_queue_instructions['systemd3'] = 'Execute the following commands to enable the timer, and monitor your journal to make sure that it is operating as scheduled.'; $lang->autoinstall = 'EasyInstall'; $lang->last_week = 'Last Week'; $lang->this_week = 'This Week'; diff --git a/modules/admin/lang/ko.php b/modules/admin/lang/ko.php index e45e4ab20..7b2bd4979 100644 --- a/modules/admin/lang/ko.php +++ b/modules/admin/lang/ko.php @@ -295,6 +295,13 @@ $lang->cmd_queue_config_keys['port'] = '포트'; $lang->cmd_queue_config_keys['user'] = '아이디'; $lang->cmd_queue_config_keys['pass'] = '암호'; $lang->cmd_queue_config_keys['dbnum'] = 'DB 번호'; +$lang->msg_queue_instructions['same_as_php'] = 'PHP를 실행하는 계정과 동일한'; +$lang->msg_queue_instructions['crontab1'] = '%s 계정으로 서버에 로그인하여 crontab -e 명령을 실행한 후, 아래의 내용을 붙여넣으십시오. (root 권한으로 실행하지 마십시오.)
    예제의 %s 디렉토리는 로그를 기록할 수 있는 경로로 변경하여 사용하십시오.'; +$lang->msg_queue_instructions['crontab2'] = '스크립트 호출 간격을 변경할 경우, 설정에 맞추어 crontab 실행 간격도 조절하여야 합니다.'; +$lang->msg_queue_instructions['webcron'] = '아래의 URL을 1분 간격 또는 아래에서 설정한 호출 간격에 맞추어 GET으로 호출하도록 합니다.
    웹크론 서비스가 방화벽이나 CDN 등에 의해 차단되지 않도록 주의하고, 정상적으로 호출되는지 서버 로그를 확인하십시오.'; +$lang->msg_queue_instructions['systemd1'] = '/etc/systemd/system/rhymix-queue.service 파일에 아래와 같은 내용을 넣습니다.'; +$lang->msg_queue_instructions['systemd2'] = '/etc/systemd/system/rhymix-queue.timer 파일에 아래와 같은 내용을 넣습니다.'; +$lang->msg_queue_instructions['systemd3'] = '아래의 명령을 실행하여 타이머를 활성화하고, 정상 작동하는지 모니터링하십시오.'; $lang->autoinstall = '쉬운 설치'; $lang->last_week = '지난주'; $lang->this_week = '이번주'; diff --git a/modules/admin/tpl/config_queue.html b/modules/admin/tpl/config_queue.html index 1e873d488..eda100a42 100644 --- a/modules/admin/tpl/config_queue.html +++ b/modules/admin/tpl/config_queue.html @@ -1,6 +1,7 @@ +
    @@ -96,6 +97,69 @@

    {$lang->cmd_queue_call_script}

    +
    + +
    + {@ + if (function_exists('posix_getpwuid') && function_exists('posix_getuid')): + $user_info = posix_getpwuid(posix_getuid()); + if (!empty($user_info['dir'])): + $user_info['dir'] .= DIRECTORY_SEPARATOR; + endif; + else: + $user_info = []; + $user_info['name'] = $lang->msg_queue_instructions['same_as_php']; + endif; + } +

    + {sprintf($lang->msg_queue_instructions['crontab1'], $user_info['name'] ?? 'PHP', $user_info['dir'] . 'logs')|noescape} +

    + 
    * * * * * php {\RX_BASEDIR}index.php common.cron >> {$user_info['dir']}logs{\DIRECTORY_SEPARATOR}queue.log 2>&1
    +

    + {$lang->msg_queue_instructions['crontab2']|noescape} +

    +
    +
    +

    + {$lang->msg_queue_instructions['webcron']|noescape} +

    + 
    {getFullUrl('')}common/scripts/cron.php?key={config('queue.key')}
    +
    +
    +

    + {$lang->msg_queue_instructions['systemd1']|noescape} +

    + 
    [Unit]
+Description=Rhymix Queue Service
+
+[Service]
+ExecStart=php {\RX_BASEDIR}index.php common.cron
+User={$user_info['name']}
    +

    + {$lang->msg_queue_instructions['systemd2']|noescape} +

    + 
    [Unit]
+Description=Rhymix Queue Timer
+
+[Timer]
+OnCalendar=*-*-* *:*:00
+Unit=rhymix-queue.service
+
+[Install]
+WantedBy=multi-user.target
    +

    + {$lang->msg_queue_instructions['systemd3']|noescape} +

    + 
    systemctl daemon-reload
+systemctl start rhymix-queue.timer
+systemctl enable rhymix-queue.timer
    +
    +
    +
    diff --git a/modules/admin/tpl/css/queue_config.scss b/modules/admin/tpl/css/queue_config.scss new file mode 100644 index 000000000..c11f8df01 --- /dev/null +++ b/modules/admin/tpl/css/queue_config.scss @@ -0,0 +1,27 @@ +.queue-script-setup { + .qss-content { + display: none; + &.active { + display: block; + border: 1px solid #ddd; + border-top: 0; + margin-top: -20px; + padding: 20px 12px 10px 12px; + border-bottom-left-radius: 4px; + border-bottom-right-radius: 4px; + margin-bottom: 20px; + } + .qss-instruction { + margin-bottom: 10px; + code { + color: #333; + border: 0; + background-color: transparent; + padding: 0 1px; + } + } + .pre { + margin-bottom: 10px; + } + } +} diff --git a/modules/admin/tpl/js/queue_config.js b/modules/admin/tpl/js/queue_config.js index 53af72f2f..f29291b52 100644 --- a/modules/admin/tpl/js/queue_config.js +++ b/modules/admin/tpl/js/queue_config.js @@ -2,7 +2,7 @@ $(function() { $("#queue_driver").on("change", function() { - var selected_driver = $(this).val(); + const selected_driver = $(this).val(); $(this).parents("section").find("div.x_control-group.hidden-by-default, p.x_help-block.hidden-by-default").each(function() { if ($(this).hasClass("show-for-" + selected_driver)) { $(this).show(); @@ -12,5 +12,22 @@ }); }).triggerHandler("change"); + $("#queue_key").on('change keyup paste', function() { + const key = encodeURIComponent(String($(this).val())); + $('.webcron-url').text($('.webcron-url').text().replace(/\?key=[a-zA-Z0-9]+/g, '?key=' + key)); + }); + + const qss = $('.queue-script-setup'); + const qss_tabs = qss.find('.qss-tabs'); + const qss_content = qss.find('.qss-content'); + qss_tabs.on('click', 'a', function(event) { + const selected_tab = $(this).data('value'); + qss_tabs.find('li').removeClass('x_active'); + $(this).parent().addClass('x_active'); + qss_content.removeClass('active'); + qss_content.filter('.' + selected_tab).addClass('active'); + event.preventDefault(); + }); + }); })(jQuery); From 61b9f57196dc5a06f5a6c8d2bc08cb415584cf11 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 23:22:42 +0900 Subject: [PATCH 07/16] Implement multiprocessing, and try to make webcron more resilient --- common/framework/Queue.php | 3 -- common/scripts/cron.php | 59 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/common/framework/Queue.php b/common/framework/Queue.php index 810d4513f..56410e13f 100644 --- a/common/framework/Queue.php +++ b/common/framework/Queue.php @@ -178,9 +178,6 @@ class Queue */ public static function process(int $timeout): void { - // Increase the time limit. This may or may not work. - set_time_limit(min(60, $timeout)); - // This part will run in a loop until timeout. $process_start_time = microtime(true); while (true) diff --git a/common/scripts/cron.php b/common/scripts/cron.php index 96e57b384..c07fabff8 100644 --- a/common/scripts/cron.php +++ b/common/scripts/cron.php @@ -22,6 +22,7 @@ else $key = (string)Context::get('key'); if (!Rhymix\Framework\Queue::checkKey($key)) { + Context::setCacheControl(0); header('HTTP/1.1 403 Forbidden'); echo "Invalid key\n"; Context::close(); @@ -29,6 +30,60 @@ else } } -// The rest of the work will be done by the Queue class. +// Get queue configuration set by the administrator. $timeout = (config('queue.interval') ?? 1) * 60; -Rhymix\Framework\Queue::process($timeout); +$process_count = config('queue.process_count') ?? 1; + +// If called over the network, try to increase the timeout. +if (PHP_SAPI !== 'cli') +{ + ignore_user_abort(true); + set_time_limit(max(60, $timeout)); +} + +// Create multiple processes if configured. +if (PHP_SAPI === 'cli' && $process_count > 1 && function_exists('pcntl_fork') && function_exists('pcntl_waitpid')) +{ + // This array will keep a dictionary of subprocesses. + $pids = []; + + // The database connection must be closed before forking. + Rhymix\Framework\DB::getInstance()->disconnect(); + Rhymix\Framework\Debug::disable(); + + // Create the required number of subprocesses. + for ($i = 0; $i < $process_count; $i++) + { + $pid = pcntl_fork(); + if ($pid > 0) + { + $pids[$pid] = true; + usleep(200000); + } + elseif ($pid == 0) + { + Rhymix\Framework\Queue::process($timeout); + exit; + } + else + { + error_log('RxQueue: could not fork!'); + exit; + } + } + + // The parent process waits for its children to finish. + while (count($pids)) + { + $pid = pcntl_waitpid(-1, $status, \WNOHANG); + if ($pid) + { + unset($pids[$pid]); + } + usleep(200000); + } +} +else +{ + Rhymix\Framework\Queue::process($timeout); +} From 0fd3b42885b455bb5472531ff879cfc3838f4e32 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 23:30:54 +0900 Subject: [PATCH 08/16] Fill in error messages related to Queue configuration --- modules/admin/controllers/systemconfig/Queue.php | 12 ++++++------ modules/admin/lang/en.php | 6 ++++++ modules/admin/lang/ko.php | 6 ++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/modules/admin/controllers/systemconfig/Queue.php b/modules/admin/controllers/systemconfig/Queue.php index 6ae53634c..4cc4b5dc0 100644 --- a/modules/admin/controllers/systemconfig/Queue.php +++ b/modules/admin/controllers/systemconfig/Queue.php @@ -60,11 +60,11 @@ class Queue extends Base $driver = trim($vars->queue_driver); if (!array_key_exists($driver, $drivers)) { - throw new Exception('1111'); + throw new Exception('msg_queue_driver_not_found'); } if ($enabled && (!$driver || $driver === 'dummy')) { - throw new Exception('2222'); + throw new Exception('msg_queue_driver_cannot_be_dummy'); } // Validate required and optional driver settings. @@ -74,7 +74,7 @@ class Queue extends Base $conf_value = trim($vars->{'queue_' . $driver . '_' . $conf_name} ?? ''); if ($conf_value === '') { - throw new Exception('3333'); + throw new Exception('msg_queue_invalid_config'); } $driver_config[$conf_name] = $conf_value === '' ? null : $conf_value; } @@ -88,21 +88,21 @@ class Queue extends Base $interval = intval($vars->queue_interval ?? 1); if ($interval < 1 || $interval > 10) { - throw new Exception('4444'); + throw new Exception('msg_queue_invalid_interval'); } // Validate the process count. $process_count = intval($vars->queue_process_count ?? 1); if ($process_count < 1 || $process_count > 10) { - throw new Exception('4444'); + throw new Exception('msg_queue_invalid_process_count'); } // Validate the key. $key = trim($vars->queue_key ?? ''); if (strlen($key) < 16 || !ctype_alnum($key)) { - throw new Exception('5555'); + throw new Exception('msg_queue_invalid_key'); } // Save system config. diff --git a/modules/admin/lang/en.php b/modules/admin/lang/en.php index 6e9618e1e..5f5afa113 100644 --- a/modules/admin/lang/en.php +++ b/modules/admin/lang/en.php @@ -306,6 +306,12 @@ $lang->msg_queue_instructions['webcron'] = 'Configure an external cron service t $lang->msg_queue_instructions['systemd1'] = 'Put the following content in /etc/systemd/system/rhymix-queue.service'; $lang->msg_queue_instructions['systemd2'] = 'Put the following content in /etc/systemd/system/rhymix-queue.timer'; $lang->msg_queue_instructions['systemd3'] = 'Execute the following commands to enable the timer, and monitor your journal to make sure that it is operating as scheduled.'; +$lang->msg_queue_driver_not_found = 'Invalid task queue driver'; +$lang->msg_queue_driver_cannot_be_dummy = 'In otder to use the task queue, you must select a driver other than "Not use"'; +$lang->msg_queue_invalid_config = 'Missing or invalid configuration for the selected queue driver.'; +$lang->msg_queue_invalid_interval = 'The calling interval must be between 1 and 10 minutes.'; +$lang->msg_queue_invalid_process_count = 'The process count must be between 1 and 10.'; +$lang->msg_queue_invalid_key = 'The webcron auth key must be at least 16 characters long, and only consist of alphanumeric characters.'; $lang->autoinstall = 'EasyInstall'; $lang->last_week = 'Last Week'; $lang->this_week = 'This Week'; diff --git a/modules/admin/lang/ko.php b/modules/admin/lang/ko.php index 7b2bd4979..f6f3ff3f8 100644 --- a/modules/admin/lang/ko.php +++ b/modules/admin/lang/ko.php @@ -302,6 +302,12 @@ $lang->msg_queue_instructions['webcron'] = '아래의 URL을 1분 간격 또는 $lang->msg_queue_instructions['systemd1'] = '/etc/systemd/system/rhymix-queue.service 파일에 아래와 같은 내용을 넣습니다.'; $lang->msg_queue_instructions['systemd2'] = '/etc/systemd/system/rhymix-queue.timer 파일에 아래와 같은 내용을 넣습니다.'; $lang->msg_queue_instructions['systemd3'] = '아래의 명령을 실행하여 타이머를 활성화하고, 정상 작동하는지 모니터링하십시오.'; +$lang->msg_queue_driver_not_found = '지원하지 않는 비동기 드라이버입니다.'; +$lang->msg_queue_driver_cannot_be_dummy = '비동기 작업을 사용하려면 "미사용" 이외의 드라이버를 선택해야 합니다.'; +$lang->msg_queue_invalid_config = '비동기 드라이버의 필수 설정이 누락되었습니다.'; +$lang->msg_queue_invalid_interval = '호출 간격은 1~10분 이내여야 합니다.'; +$lang->msg_queue_invalid_process_count = '프로세스 갯수는 1~10개 이내여야 합니다.'; +$lang->msg_queue_invalid_key = '웹크론 인증키는 16자 이상으로, 알파벳 대소문자와 숫자만으로 이루어져야 합니다.'; $lang->autoinstall = '쉬운 설치'; $lang->last_week = '지난주'; $lang->this_week = '이번주'; From f6a458f648cc911ed29519307e52fd702bceb155 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 23:43:04 +0900 Subject: [PATCH 09/16] Add RXQUEUE_CRON constant to distinguish when cron is running --- common/scripts/cron.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/scripts/cron.php b/common/scripts/cron.php index c07fabff8..563f71c70 100644 --- a/common/scripts/cron.php +++ b/common/scripts/cron.php @@ -6,14 +6,16 @@ * Unlike other scripts provided with Rhymix, it can be called * both on the command line and over the network. */ +define('RXQUEUE_CRON', true); +// If called on the CLI, run additional checks. if (PHP_SAPI === 'cli') { require_once __DIR__ . '/common.php'; } else { - // If called over the network, bypass CLI checks. + // If called over the network, load Rhymix directly. chdir(dirname(dirname(__DIR__))); require_once dirname(__DIR__) . '/autoload.php'; Context::init(); From a1abf5016f6830e54809c2a88252f61400180495 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 23:46:25 +0900 Subject: [PATCH 10/16] Integrate Mail and SMS with Queue --- common/framework/Mail.php | 18 ++++++++++++++++++ common/framework/SMS.php | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/common/framework/Mail.php b/common/framework/Mail.php index 7502c6df9..bcf9b62a3 100644 --- a/common/framework/Mail.php +++ b/common/framework/Mail.php @@ -560,6 +560,13 @@ class Mail */ public function send(): bool { + // If queue is enabled, send asynchronously. + if (config('queue.enabled') && !defined('RXQUEUE_CRON')) + { + Queue::addTask(self::class . '::' . 'sendAsync', $this); + return true; + } + // Get caller information. $backtrace = debug_backtrace(0); if(count($backtrace) && isset($backtrace[0]['file'])) @@ -600,6 +607,17 @@ class Mail return $this->sent; } + /** + * Send an email asynchronously (for Queue integration). + * + * @param self $mail + * @return void + */ + public static function sendAsync(self $mail): void + { + $mail->send(); + } + /** * Check if the message was sent. * diff --git a/common/framework/SMS.php b/common/framework/SMS.php index 0d347bc83..4bb65fd60 100644 --- a/common/framework/SMS.php +++ b/common/framework/SMS.php @@ -511,6 +511,13 @@ class SMS */ public function send(): bool { + // If queue is enabled, send asynchronously. + if (config('queue.enabled') && !defined('RXQUEUE_CRON')) + { + Queue::addTask(self::class . '::' . 'sendAsync', $this); + return true; + } + // Get caller information. $backtrace = debug_backtrace(0); if(count($backtrace) && isset($backtrace[0]['file'])) @@ -566,6 +573,17 @@ class SMS return $this->sent; } + /** + * Send an SMS asynchronously (for Queue integration). + * + * @param self $sms + * @return void + */ + public static function sendAsync(self $sms): void + { + $sms->send(); + } + /** * Check if the message was sent. * From b4e7d4aaad2ecd7f814d94ccaf0692582f22b370 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Thu, 10 Oct 2024 23:54:53 +0900 Subject: [PATCH 11/16] Integrate Push with Queue --- common/framework/Push.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/common/framework/Push.php b/common/framework/Push.php index 547a2295e..bc37cd2c7 100644 --- a/common/framework/Push.php +++ b/common/framework/Push.php @@ -402,6 +402,13 @@ class Push */ public function send(): bool { + // If queue is enabled, send asynchronously. + if (config('queue.enabled') && !defined('RXQUEUE_CRON')) + { + Queue::addTask(self::class . '::' . 'sendAsync', $this); + return true; + } + // Get caller information. $backtrace = debug_backtrace(0); if(count($backtrace) && isset($backtrace[0]['file'])) @@ -464,6 +471,17 @@ class Push return $this->sent > 0 ? true : false; } + /** + * Send asynchronously (for Queue integration). + * + * @param self $sms + * @return void + */ + public static function sendAsync(self $push): void + { + $push->send(); + } + /** * Get the device token * From 4b0b485a135c6e3f17a6d9ec8e2eabba5d900b36 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Fri, 11 Oct 2024 23:19:53 +0900 Subject: [PATCH 12/16] Add basic unit tests for Queue class using Dummy driver --- common/framework/drivers/queue/dummy.php | 14 +++++++++++++- tests/unit/framework/QueueTest.php | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/unit/framework/QueueTest.php diff --git a/common/framework/drivers/queue/dummy.php b/common/framework/drivers/queue/dummy.php index c39d10947..984bff83e 100644 --- a/common/framework/drivers/queue/dummy.php +++ b/common/framework/drivers/queue/dummy.php @@ -9,6 +9,11 @@ use Rhymix\Framework\Drivers\QueueInterface; */ class Dummy implements QueueInterface { + /** + * Dummy queue for testing. + */ + protected $_dummy_queue; + /** * Create a new instance of the current Queue driver, using the given settings. * @@ -80,6 +85,11 @@ class Dummy implements QueueInterface */ public function addTask(string $handler, ?object $args = null, ?object $options = null): int { + $this->_dummy_queue = (object)[ + 'handler' => $handler, + 'args' => $args, + 'options' => $options, + ]; return 0; } @@ -91,6 +101,8 @@ class Dummy implements QueueInterface */ public function getTask(int $blocking = 0): ?object { - return null; + $result = $this->_dummy_queue; + $this->_dummy_queue = null; + return $result; } } diff --git a/tests/unit/framework/QueueTest.php b/tests/unit/framework/QueueTest.php new file mode 100644 index 000000000..5019a7da3 --- /dev/null +++ b/tests/unit/framework/QueueTest.php @@ -0,0 +1,23 @@ + 'bar']; + $options = (object)['key' => 'val']; + + Rhymix\Framework\Queue::addTask($handler, $args, $options); + + $output = Rhymix\Framework\Queue::getTask(); + $this->assertEquals('myfunc', $output->handler); + $this->assertEquals('bar', $output->args->foo); + $this->assertEquals('val', $output->options->key); + + $output = Rhymix\Framework\Queue::getTask(); + $this->assertNull($output); + } +} From 29837bdf4c21b81efc53a8bdf4f220e5ee6828e1 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Fri, 11 Oct 2024 23:38:29 +0900 Subject: [PATCH 13/16] Fix return type of QueueInterface::getInstance() --- common/framework/drivers/QueueInterface.php | 4 ++-- common/framework/drivers/queue/db.php | 4 ++-- common/framework/drivers/queue/dummy.php | 4 ++-- common/framework/drivers/queue/redis.php | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/framework/drivers/QueueInterface.php b/common/framework/drivers/QueueInterface.php index f3c5697f5..25acb16fb 100644 --- a/common/framework/drivers/QueueInterface.php +++ b/common/framework/drivers/QueueInterface.php @@ -11,9 +11,9 @@ interface QueueInterface * Create a new instance of the current Queue driver, using the given settings. * * @param array $config - * @return void + * @return QueueInterface */ - public static function getInstance(array $config): self; + public static function getInstance(array $config): QueueInterface; /** * Get the human-readable name of this Queue driver. diff --git a/common/framework/drivers/queue/db.php b/common/framework/drivers/queue/db.php index f92e5614f..f393c7de4 100644 --- a/common/framework/drivers/queue/db.php +++ b/common/framework/drivers/queue/db.php @@ -14,9 +14,9 @@ class DB implements QueueInterface * Create a new instance of the current Queue driver, using the given settings. * * @param array $config - * @return void + * @return QueueInterface */ - public static function getInstance(array $config): self + public static function getInstance(array $config): QueueInterface { return new self($config); } diff --git a/common/framework/drivers/queue/dummy.php b/common/framework/drivers/queue/dummy.php index 984bff83e..2996f7c98 100644 --- a/common/framework/drivers/queue/dummy.php +++ b/common/framework/drivers/queue/dummy.php @@ -18,9 +18,9 @@ class Dummy implements QueueInterface * Create a new instance of the current Queue driver, using the given settings. * * @param array $config - * @return void + * @return QueueInterface */ - public static function getInstance(array $config): self + public static function getInstance(array $config): QueueInterface { return new self($config); } diff --git a/common/framework/drivers/queue/redis.php b/common/framework/drivers/queue/redis.php index c5d0c17bc..ac6eb3074 100644 --- a/common/framework/drivers/queue/redis.php +++ b/common/framework/drivers/queue/redis.php @@ -19,9 +19,9 @@ class Redis implements QueueInterface * Create a new instance of the current Queue driver, using the given settings. * * @param array $config - * @return void + * @return QueueInterface */ - public static function getInstance(array $config): self + public static function getInstance(array $config): QueueInterface { return new self($config); } From f1eee10791b9ed30348bd22e279b072b2f1e6232 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Sat, 12 Oct 2024 01:21:43 +0900 Subject: [PATCH 14/16] Remove unnecessary "lock" column from task_queue table --- modules/module/schemas/task_queue.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/module/schemas/task_queue.xml b/modules/module/schemas/task_queue.xml index 6e991a700..73e18ab33 100644 --- a/modules/module/schemas/task_queue.xml +++ b/modules/module/schemas/task_queue.xml @@ -3,6 +3,5 @@ - From 0ee9747a22ae7a5540f217c46807093807b749b2 Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Sat, 12 Oct 2024 01:22:04 +0900 Subject: [PATCH 15/16] Use SELECT ... FOR UPDATE to guard against race conditions in DB queue driver --- common/framework/drivers/queue/db.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/framework/drivers/queue/db.php b/common/framework/drivers/queue/db.php index f393c7de4..fbb312e7d 100644 --- a/common/framework/drivers/queue/db.php +++ b/common/framework/drivers/queue/db.php @@ -96,7 +96,8 @@ class DB implements QueueInterface public function getTask(int $blocking = 0): ?object { $oDB = RFDB::getInstance(); - $stmt = $oDB->query('SELECT * FROM task_queue ORDER BY id LIMIT 1'); + $oDB->beginTransaction(); + $stmt = $oDB->query('SELECT * FROM task_queue ORDER BY id LIMIT 1 FOR UPDATE'); $result = $stmt->fetchObject(); $stmt->closeCursor(); @@ -104,6 +105,7 @@ class DB implements QueueInterface { $stmt = $oDB->prepare('DELETE FROM task_queue WHERE id = ?'); $stmt->execute([$result->id]); + $oDB->commit(); $result->args = unserialize($result->args); $result->options = unserialize($result->options); @@ -111,6 +113,7 @@ class DB implements QueueInterface } else { + $oDB->commit(); return null; } } From 36af489b1585f328971b817a7bfd8a0567a4dbec Mon Sep 17 00:00:00 2001 From: Kijin Sung Date: Sat, 12 Oct 2024 01:35:13 +0900 Subject: [PATCH 16/16] Validate queue driver configuration before saving --- common/framework/drivers/QueueInterface.php | 8 +++++ common/framework/drivers/queue/db.php | 11 ++++++ common/framework/drivers/queue/dummy.php | 11 ++++++ common/framework/drivers/queue/redis.php | 36 +++++++++++++++++-- .../admin/controllers/systemconfig/Queue.php | 14 +++++++- modules/admin/lang/en.php | 3 +- modules/admin/lang/ko.php | 3 +- 7 files changed, 80 insertions(+), 6 deletions(-) diff --git a/common/framework/drivers/QueueInterface.php b/common/framework/drivers/QueueInterface.php index 25acb16fb..71d71a452 100644 --- a/common/framework/drivers/QueueInterface.php +++ b/common/framework/drivers/QueueInterface.php @@ -43,6 +43,14 @@ interface QueueInterface */ public static function isSupported(): bool; + /** + * Validate driver configuration. + * + * @param mixed $config + * @return bool + */ + public static function validateConfig($config): bool; + /** * Add a task. * diff --git a/common/framework/drivers/queue/db.php b/common/framework/drivers/queue/db.php index fbb312e7d..c45eedc5d 100644 --- a/common/framework/drivers/queue/db.php +++ b/common/framework/drivers/queue/db.php @@ -61,6 +61,17 @@ class DB implements QueueInterface return true; } + /** + * Validate driver configuration. + * + * @param mixed $config + * @return bool + */ + public static function validateConfig($config): bool + { + return true; + } + /** * Constructor. * diff --git a/common/framework/drivers/queue/dummy.php b/common/framework/drivers/queue/dummy.php index 2996f7c98..0abb74792 100644 --- a/common/framework/drivers/queue/dummy.php +++ b/common/framework/drivers/queue/dummy.php @@ -65,6 +65,17 @@ class Dummy implements QueueInterface return true; } + /** + * Validate driver configuration. + * + * @param mixed $config + * @return bool + */ + public static function validateConfig($config): bool + { + return true; + } + /** * Constructor. * diff --git a/common/framework/drivers/queue/redis.php b/common/framework/drivers/queue/redis.php index ac6eb3074..58c122184 100644 --- a/common/framework/drivers/queue/redis.php +++ b/common/framework/drivers/queue/redis.php @@ -66,6 +66,37 @@ class Redis implements QueueInterface return class_exists('\\Redis'); } + /** + * Validate driver configuration. + * + * @param mixed $config + * @return bool + */ + public static function validateConfig($config): bool + { + try + { + $test = new \Redis; + $test->connect($config['host'], $config['port'] ?? 6379); + if (isset($config['user']) || isset($config['pass'])) + { + $auth = []; + if (isset($config['user']) && $config['user']) $auth[] = $config['user']; + if (isset($config['pass']) && $config['pass']) $auth[] = $config['pass']; + $test->auth(count($auth) > 1 ? $auth : $auth[0]); + } + if (isset($config['dbnum'])) + { + $test->select(intval($config['dbnum'])); + } + return true; + } + catch (\Throwable $th) + { + return false; + } + } + /** * Constructor. * @@ -77,18 +108,17 @@ class Redis implements QueueInterface { $this->_conn = new \Redis; $this->_conn->connect($config['host'], $config['port'] ?? 6379); - if(isset($config['user']) || isset($config['pass'])) + if (isset($config['user']) || isset($config['pass'])) { $auth = []; if (isset($config['user']) && $config['user']) $auth[] = $config['user']; if (isset($config['pass']) && $config['pass']) $auth[] = $config['pass']; $this->_conn->auth(count($auth) > 1 ? $auth : $auth[0]); } - if(isset($config['dbnum'])) + if (isset($config['dbnum'])) { $this->_conn->select(intval($config['dbnum'])); } - $this->_key = 'rxQueue_' . substr(hash_hmac('sha1', 'rxQueue_', config('crypto.authentication_key')), 0, 24); } catch (\RedisException $e) diff --git a/modules/admin/controllers/systemconfig/Queue.php b/modules/admin/controllers/systemconfig/Queue.php index 4cc4b5dc0..1422dbbc5 100644 --- a/modules/admin/controllers/systemconfig/Queue.php +++ b/modules/admin/controllers/systemconfig/Queue.php @@ -68,7 +68,7 @@ class Queue extends Base } // Validate required and optional driver settings. - $driver_config = array(); + $driver_config = []; foreach ($drivers[$driver]['required'] as $conf_name) { $conf_value = trim($vars->{'queue_' . $driver . '_' . $conf_name} ?? ''); @@ -105,6 +105,18 @@ class Queue extends Base throw new Exception('msg_queue_invalid_key'); } + // Validate actual operation of the driver. + $driver_class = '\\Rhymix\\Framework\\Drivers\\Queue\\' . $driver; + if (!class_exists($driver_class) || !$driver_class::isSupported()) + { + throw new Exception('msg_queue_driver_not_found'); + } + if (!$driver_class::validateConfig($driver_config)) + { + throw new Exception('msg_queue_driver_not_usable'); + } + + // Save system config. Config::set("queue.enabled", $enabled); Config::set("queue.driver", $driver); diff --git a/modules/admin/lang/en.php b/modules/admin/lang/en.php index 5f5afa113..a2ff0abbb 100644 --- a/modules/admin/lang/en.php +++ b/modules/admin/lang/en.php @@ -306,7 +306,8 @@ $lang->msg_queue_instructions['webcron'] = 'Configure an external cron service t $lang->msg_queue_instructions['systemd1'] = 'Put the following content in /etc/systemd/system/rhymix-queue.service'; $lang->msg_queue_instructions['systemd2'] = 'Put the following content in /etc/systemd/system/rhymix-queue.timer'; $lang->msg_queue_instructions['systemd3'] = 'Execute the following commands to enable the timer, and monitor your journal to make sure that it is operating as scheduled.'; -$lang->msg_queue_driver_not_found = 'Invalid task queue driver'; +$lang->msg_queue_driver_not_found = 'The selected task queue driver is not supported on this server.'; +$lang->msg_queue_driver_not_usable = 'The selected task queue driver failed to initialize using the configuration values you entered.'; $lang->msg_queue_driver_cannot_be_dummy = 'In otder to use the task queue, you must select a driver other than "Not use"'; $lang->msg_queue_invalid_config = 'Missing or invalid configuration for the selected queue driver.'; $lang->msg_queue_invalid_interval = 'The calling interval must be between 1 and 10 minutes.'; diff --git a/modules/admin/lang/ko.php b/modules/admin/lang/ko.php index f6f3ff3f8..7fd335612 100644 --- a/modules/admin/lang/ko.php +++ b/modules/admin/lang/ko.php @@ -302,7 +302,8 @@ $lang->msg_queue_instructions['webcron'] = '아래의 URL을 1분 간격 또는 $lang->msg_queue_instructions['systemd1'] = '/etc/systemd/system/rhymix-queue.service 파일에 아래와 같은 내용을 넣습니다.'; $lang->msg_queue_instructions['systemd2'] = '/etc/systemd/system/rhymix-queue.timer 파일에 아래와 같은 내용을 넣습니다.'; $lang->msg_queue_instructions['systemd3'] = '아래의 명령을 실행하여 타이머를 활성화하고, 정상 작동하는지 모니터링하십시오.'; -$lang->msg_queue_driver_not_found = '지원하지 않는 비동기 드라이버입니다.'; +$lang->msg_queue_driver_not_found = '이 서버에서 지원하지 않는 비동기 드라이버입니다.'; +$lang->msg_queue_driver_not_usable = '입력하신 정보로 비동기 드라이버와 연동하는 데 실패했습니다. 드라이버 설정을 확인해 주십시오.'; $lang->msg_queue_driver_cannot_be_dummy = '비동기 작업을 사용하려면 "미사용" 이외의 드라이버를 선택해야 합니다.'; $lang->msg_queue_invalid_config = '비동기 드라이버의 필수 설정이 누락되었습니다.'; $lang->msg_queue_invalid_interval = '호출 간격은 1~10분 이내여야 합니다.';